#!/usr/bin/env bash
#
# 文 件 名: multi-install-k8s.sh
# 文件编码: UTF-8
# 编程语言: Shell Script(Bash 4.x/5.x)
# 编程工具: VSCode/Goland
# 显示语言: 简体中文
# 脚本编写：杨芳超
# 联系邮箱: i-up@qq.com
# 问题反馈: https://seeyon.ren
#

# set -x

# 脚本功能
# 1. 在基于 amd64/arm64 架构的 linux 上在线/离线 部署 k8s 集群
# 2. 建议使用最小化安装的纯净操作系统
## redhat 系列: RHEL/Alma/Rocky/OL/Anolis/CentOS 7/8/9 | UOS V20 1060a/1060e | Kylin V10 | OpenEuler 22.03
## debian 系列: Debian 11/12 | Ubuntu 20.04/22.04

# 注意事项
# 1、必须在 k8s 集群 master 节点的第一个节点上运行
# 2、支持 containerd 和 docker 容器运行时 强烈建议使用 containerd 作为容器运行时
# 3、etcd 集群为堆叠架构部署在 master 节点上
# 4、软件源需要能正常使用 安装依赖和 keepalived/haproxy 时需要用到
# 5、经过实测并发版本 3 + 3/5 个节点的情况下 速度提升不是很明显 大约提速 3-5 分钟左右 意义不大 所以就不发布并发版本了
# 6、部署集群前，请检查所有集群节点的时间差不超过3毫秒。

# 离线导入时的镜像文件列表 务必确保各版本匹配 否则部署失败 注意平台架构支持
# 以下是经过验证的版本: 1.24/1.25/1.26/1.27/1.28/1.29/1.30

# 镜像处理策略 只能是 1|2 1=在线拉取 2=离线导入 离线镜像文件目录下 不要放多余的版本或不支持的架构的版本
# 在线拉取: 使用官方镜像仓库、国内镜像仓库、本地镜像仓库
# 离线导入: 自动导入官方 tar 容器镜像 无需联网 启用此项时 将自动调整仓库为官方地址
imagePolicy="2"

# k8s 集群 vip 主机名
vipHostName="k8s-vip"
# k8s 集群 vip 地址
vipAddr="192.168.100.150"
# k8s 集群 vip 端口
vipPort="9443"
# k8s 集群 vip 域名
vipDomain="k8s.oso.plus"

# 容器运行时名称 只能是 containerd|docker|podman 暂不支持 podman
# 如果容器运行时使用 docker 则只支持在线安装 需保证 docker-ce 的仓库可正常访问
criRuntime="containerd"
# 容器运行时 CRI 套接字
containerdSock="unix:///run/containerd/containerd.sock"
criDockerdSock="unix:///var/run/cri-dockerd.sock"
# dockerSock="unix:///var/run/docker.sock"

# k8s 集群 镜像仓库 官方仓库为 registry.k8s.io 当 imagePolicy="2" 即使用官方离线镜像时 脚本将强制修改为官方地址 不用带协议
imgRegAddr="r.oso.plus/k8s"
# imgRegAddr="registry.k8s.io"
# imgRegAddr="registry.cn-hangzhou.aliyuncs.com/google_containers"

# 是否添加镜像仓库认证信息 1=添加
addRegAuth="1"
# 待添加的镜像仓库地址 不用带协议和二级目录 此地址为独立的地址 与 imgRegAddr 无关
regAddr="r.oso.plus"
# 镜像仓库登录用户名和密码 用于拉取镜像 registry 例如 harbor 的账号和密码
regUser="admin"
# 镜像仓库用户密码
regPass="i4Seeyon"
# 镜像仓库 访问协议 只能是 https|http
regProtocol="https"
# 镜像仓库 CA（证书颁发机构）证书的路径 用于验证服务器证书
regCaPath="/etc/containerd/certs.d/r.oso.plus/ca.cer"
# 镜像仓库 客户端证书的路径 用于在TLS连接中身份验证
regCrtPath="/etc/containerd/certs.d/r.oso.plus/r.oso.plus.cer"
# 镜像仓库 客户端私钥的路径 用于在TLS连接中加密通信
regKeyPath="/etc/containerd/certs.d/r.oso.plus/r.oso.plus.key"
# k8s 沙箱版本 脚本历史遗留参数 已调整为脚本自动获取
# pauseVer="3.9"
# k8s 集群版本 必须是 1.24.0+ 脚本基于此处设置的版本进行部署 部署过程中将对实际部署的版本进行对比 请务必确保版本一致
k8sVer="1.31.0"
# k8s 集群名称
clusterName="k8s-cluster"
# k8s 集群 dns 域
dnsDomain="cluster.local"
# k8s 集群 Pod 子网的 CIDR
podSubnet="10.244.0.0/16"
# k8s 集群 Service 子网的 CIDR
serviceSubnet="10.96.0.0/12"
# k8s 集群 控制平面 节点端口
bindPort="6443"
# k8s 集群 控制平面/master 节点列表 格式：ip 节点名
# 要求: 至少 1 个 master 节点 为了避免脑裂问题 强制按照奇数递增 例如 1、3、5、7、9 个 master 节点
# 注意：k8sNodesMaster 数组中的第一个元素将作为 控制平面/master 的第一个节点（init cluster 时的 localAPIEndpoint.advertiseAddress）
k8sNodesMaster=(
    "192.168.100.151 k8s-m1"
    "192.168.100.152 k8s-m2"
    "192.168.100.153 k8s-m3"
)
# k8s 集群 工作节点/worker 节点列表 格式：ip 节点名
# 要求: 至少 1 个 worker 节点 节点递增无特殊要求 例如 1、2、3、4、5 个 worker 节点
k8sNodesWorker=(
    "192.168.100.154 k8s-n1"
    "192.168.100.155 k8s-n2"
    "192.168.100.156 k8s-n3"
)

# calico 版本 - 版本号必须对应 template 目录下的 calico 模板文件 模板文件名格式 calico-<version>.yaml
# 当遇到最新版的 calico 与云平台存在不兼容的情况时 可尝试使用 calico v3.26.4 的版本并对应调整部署脚本中的 calico 版本号。
calicoVer="3.28.2"
# calico 镜像仓库 官方仓库地址是 docker.io/calico 当使用官方离线镜像 即 imagePolicy="2" 时 脚本将强制修改为官方地址
calicoRegAddr="r.oso.plus/calico"
# calicoRegAddr="docker.io/calico"
# calico ipv4 子网 一般情况下设置为与 podSubnet 相同 预留参数 暂不支持修改 直接使用 podSubnet 设置的值
# calicoIpv4poolCidrValue="10.244.0.0/16"
# calico 本地网卡名 - 预留 当前为脚本自动判断
# calicoIpAutodetectionMethodValue="interface=ens33"
# 是否关闭 ipip 模式 1=关闭 当设置为 1 时关闭calico的ipip模式 适用于部分平台不支持ipip的场景
calicoDisableIPIP=""

# 所有集群节点的服务器连接信息 要求端口一致,账号和密码一致，且所有节点都可以互相访问
sshPort="22"
sshUser="root"
sshPWD="110110"
# sshPWD="520@Seeyon"
# sshPWD="i+Seeyon=520"

# 二进制软件的安装根目录 不建议修改
# 脚本会在此目录下自动创建 bin 和 sbin 两个目录 并把二进制文件安装到 bin 或 sbin 目录下
installRootDir="/usr/local"

# cni-plugins 安装目录 默认使用官方软件的目录结构 不要修改此参数 因为脚本没有对相关的配置文件做对应修改
cniPluginsInstallDir="/opt/cni"

# 进行脚本需求环境及依赖检查时 是否自动安装依赖 1=是 需要确保软件更新源可用
installDepend="1"

# 自动设置免密登录 1=是 当前部署脚本不需要设置免密自动登录 开启不开启都无所谓
setNoPwdLogin=""

# 时区设置
osTimeZone="Asia/Shanghai"

# 目标主机上保存推送必需文件的目录
pushToDir="/seeyon/scripts"

# 安装软件时 如果检测到软件已经安装 则设定安装动作 1=覆盖安装/replace 2=跳过安装/skip
# 对应函数 chk_install 接受数组参数
actOnExist="1"

# 安装软件时 如果服务正在运行 是否自动关闭并覆盖安装 1=自动关闭服务 注意：如果启用此参数 配置文件也会被一起覆盖
# 对应函数 check_srv_status 和 auto_shutdown_srv 接受变量参数
autoStopSrv="1"

# 是否创建用于镜像仓库的 secret
addSecret=""
# 作用于 secret 的命名空间名称 注意：脚本将自动创建此命名空间
secretNS="seeyon-prod"
# Secret 名称 例如 harbor
secretName="harbor"
# 镜像仓库的用户
dockerUser="seeyon"
# 镜像仓库的密码
dockerPass="i4Seeyon"
# 镜像仓库的地址
dockerServer="r.oso.plus"

# master 节点部署完成后 是否添加 自定义kubectl命令的别名环境变量到当前用户的 .bashrc 文件中 1=添加
addAlias=""
# 添加的别名的命名空间
aliasNS="seeyon-prod"
# 别名命令列表 一行一条 双引号括起来 别名命令使用单引号括起来 请按需调整
aliasContent=(
    "alias k='kubectl '"
    "alias kglog='kubectl -n ${aliasNS} logs -f '"
    "alias kgdescribe='kubectl -n ${aliasNS} describe pod '"
    "alias kgpod='kubectl -n ${aliasNS} get pod '"
    "alias kgdep='kubectl -n ${aliasNS} get deployment '"
    "alias kdeldep='kubectl -n ${aliasNS} delete deploy '"
    "alias krestartdep='kubectl -n ${aliasNS} rollout restart deployment '"
)

# 指定 containerd 默认的数据存储目录和运行时状态存储目录
containerdDataDir="/data/containerd"
# 指定 kubelet 的数据存储根目录
kubeletDataDir="/data/kubelet"
# 指定 etcd 的数据存储目录
etcdDataDir="/data/etcd"

# 是否安装 helm 1=是
installHelm="1"

# 是否安装 metrics-server 1=是 原计划把 kuboard 一起装上 想想还是独立装一套 kuboard 的好 就只装这个了
installMetricsServer="1"
metricsServerRegAddr="r.oso.plus/metrics-server"
# metricsServerRegAddr="registry.k8s.io/metrics-server" # 官方
metricsServerVer="0.7.1"

# 是否尝试在脚本所在目录下解压相关部署文件压缩包 1=是
tryUnPKG="1"

# 自定义颜色函数
function oput() {
    # 功能: 自定义颜色并打印标准输出
    # 例子: oput red "系统依赖检查未通过"
    local red_color green_color yellow_color blue_color pink_color white_blue down_blue flash_red res
    # 红色
    red_color='\E[1;31m'
    # 绿色
    green_color='\E[1;32m'
    # 黄色
    yellow_color='\E[1;33m'
    # 蓝色
    blue_color='\E[1;34m'
    # 紫色
    pink_color='\E[1;35m'
    # 白底蓝字
    white_blue='\E[47;34m'
    # 下划线+蓝色
    down_blue='\E[4;36m'
    # 红闪
    flash_red='\E[5;31m'
    # 重置
    res='\E[0m'

    # 限制传入参数必须为2个 否则退出
    if [ $# -ne 2 ]; then echo "Usage $0 {red|yellow|blue|green|pink|wb|db|fr} content"; exit; fi
    case "$1" in
        red | RED ) echo -e "${red_color}$2${res}"; ;;
        yellow | YELLOW ) echo -e "${yellow_color}$2${res}"; ;;
        green | GREEN ) echo -e "${green_color}$2${res}"; ;;
        blue | BLUE ) echo -e "${blue_color}$2${res}"; ;;
        pink | PINK ) echo -e "${pink_color}$2${res}"; ;;
        wb | WB ) echo -e "${white_blue}$2${res}"; ;;
        db | DB ) echo -e "${down_blue}$2${res}"; ;;
        fr | FR ) echo -e "${flash_red}$2${res}"; ;;
        * ) echo -e "请指定一个颜色代码：{red|yellow|blue|green|pink|wb|db|fr}"; ;;
    esac
}

if [ "$(uname)" != "Linux" ]; then echo "Only Linux operating system is supported." && exit; fi
# 脚本目录 -- 与工作目录有区别
sDir="$(dirname "$(readlink -f "$0")")"
# 脚本文件
sFile="$(basename "$0")"
# 脚本bin文件根目录
sBinRootDir="${sDir}/bin"
# 脚本功能
sDesc="在基于 amd64/arm64 架构的 linux 上在线/离线 部署 k8s 集群"
# 支持环境
sEnv="amd64/arm64 + linux + bash 4.x/5.x"
# 脚本用法
sUsage="bash ${sFile} [p|i|r|v]"
# Log 目录
sLogDir="${sDir}/logs"
if ! mkdir -p "${sLogDir:?}"; then oput red "Failed to create script log directory" && exit; fi
# Log 文件
sLog="${sLogDir}/script.log"
# 生成并清空脚本运行日志
touch "${sLog}" && true >"${sLog}"
# 写入当前系统时间和脚本文件名到脚本日志
(echo; echo "$(date '+%Y-%m-%d %H:%M:%S') from ${sDir}/${sFile}"; echo;) >> "${sLog}" 2>&1
# 创建时间
sCreate="2023-12-01 15:25:28"
# 更新时间
sUpdate="2024-07-15 23:35:32"
# 更新内容
## 1、修复 脚本预置目录创建错误
## 2、添加 calico 模板的 ipip 关闭开关 - 便于在不支持ipip模式的环境下使用
## 3、支持 开启自动解压7z部署资源包时优先查找和使用 k8s-criRuntime-高版本 的7z包

# 语言环境
export LANG=zh_CN.UTF-8
# 需要管理员身份运行
if [ "$(id -un)" != "root" ]; then echo; oput red "please use 'sudo bash ${sFile}' to run script."; echo; exit; fi
# 检测系统是否支持 systemd 服务管理 若不支持则退出脚本
if ! command -v systemctl &>/dev/null; then oput red "systemd is not supported by the current OS" && exit; fi
# 切换至脚本所在目录
if ! cd "${sDir}"; then oput red "directory switching failed: ${sDir}" && exit; fi
# 主机名
hostName="$(hostname)"
# IP地址 多IP时 只取第一个IP 自动获取 如有需要请手动指定 如 hostAddr="192.168.100.111"
hostAddr="$(ip addr | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}' | grep -Ev '127.0.0|0.0.0|.255$' | head -1)"
# 获取客机IP地址 注: alpine 上who命令参数无效
fromAddr="$(who -u am i 2>/dev/null | awk '{print $NF}'| sed -e 's/[a-zA-Z:()]//g')"
# 获取不到时 一般是本地登录(use tty or pts ?)
if [ "${fromAddr}" = "" ]; then fromAddr="${hostAddr}"; fi
# 平台架构 限制为 x86_64/amd64 | aarch64/arm64
case "$(arch)" in
    "x86_64" | "amd64" ) archType="amd64"; ;;
    "arm64" | "aarch64" ) archType="arm64"; ;;
    * ) echo; oput red "the current $(arch) is not supported"; echo; exit; ;;
esac
# 系统内核
osKernel="$(uname -r)"
# glibc 版本
osLibcVer="$(ldd --version | head -n 1 | awk '{print $(NF-0)}')"
# CPU型号
cpuModel="$(export LANG=C; lscpu | awk -F ':' '/^Model name/{gsub(/^[ \t]+|[ \t]+$/,"",$2); print $2}')"
if [ "${cpuModel}" = "" ]; then cpuModel="unrecognized"; fi
# CPU 核数
cpuCount=$(grep 'processor' /proc/cpuinfo | sort | uniq | wc -l)
# 内存大小
memSize="$(awk '/^MemTotal/{print int($2/1024/1024+0.5)}' /proc/meminfo)"
# 磁盘大小
diskSize="$(lsblk -b | grep 'disk' | awk '{total+=$4/1024/1024/1024}END{print total}')"

# ************** 以下是脚本自带目录 不要随意修改 ***************

# 设置环境变量PATH为脚本预置的 bin 目录
sBinDir="${sBinRootDir}/${archType}"
# 安装文件的存放目录
softwareDir="${sDir}/software/${archType}"
# 镜像文件的存放目录
imgDir="${sDir}/images/${archType}"
# 模板文件的存放目录
templateDir="${sDir}/template"
# 脚本文档的存放目录
docDir="${sDir}/document"
# 证书文件的存放目录
certsDir="${sDir}/certs"
# k8s集群初始化日志文件 用于提取k8s集群初始化日志中的join信息
initLog="${sLogDir}/k8s-init.log"
# k8s节点加入的日志文件
joinLog="${sLogDir}/k8s-join.log"

# ************** 以上是脚本自带目录 不要随意修改 ***************

if [ -d "${sBinDir}" ]; then
    chmod -R +x "${sBinDir}"
    export PATH=${sBinDir}:/sbin:/bin:/usr/sbin:/usr/bin
else
    export PATH=/sbin:/bin:/usr/sbin:/usr/bin
fi

# OS 家族判断 脚本只支持基于 redhat/debian 的主流服务器版操作系统
if command -v yum &>/dev/null; then
    osFamily="redhat"
    osDCMD="yum"
    osPCMD="rpm"
    osRelease="$(cat /etc/system-release)"
elif command -v apt &>/dev/null; then
    osFamily="debian"
    osDCMD="apt"
    osPCMD="dpkg"
    osRelease="$(grep -E 'PRETTY_NAME' /etc/os-release | awk -F '=' '{print $2}' | tr -d '"')"
else
    echo; oput red "the current $(uname) is not supported. only redhat/debian family are supported"; echo; exit
fi

# gossh 参数选项 for gossh-andesli
gOpts="-logpath ${sLogDir} -l debug -s -f -P ${sshPort} -u ${sshUser} -p ${sshPWD}"

# 定义和设置全局配置文件路径
function def_etc_file() {
    # 软件安装根目录
    if [ ! -d "${installRootDir}" ]; then mkdir -p "${installRootDir}"/{bin,sbin}; fi
    # cni-plugins 安装目录
    if [ ! -d "${cniPluginsInstallDir}" ]; then mkdir -p "${cniPluginsInstallDir}"/{bin,net.d}; fi

    # 相关文档存放目录
    if [ ! -d "${docDir}" ]; then mkdir -p "${docDir}"; fi
    # 安装文件存放目录
    if [ ! -d "${softwareDir}/${archType}" ]; then mkdir -p "${softwareDir}/${archType}"; fi
    # 离线镜像存放目录
    if [ ! -d "${imgDir}/${archType}" ]; then mkdir -p "${imgDir}/${archType}"; fi
    # 仓库证书存放目录
    if [ ! -d "${certsDir}" ]; then mkdir -p "${certsDir}"; fi
    # 模板文件存放目录
    if [ ! -d "${templateDir}" ]; then mkdir -p "${templateDir}"; fi

    # 追加环境变量 避免脚本内执行命令异常
    local addVars
    addVars="${installRootDir}/bin:${installRootDir}/sbin:${cniPluginsInstallDir}/bin"
    export PATH="${addVars}:/usr/local/bin:/usr/local/sbin:$PATH"

    # crictl 配置文件
    etcYamlCrictl="/etc/crictl.yaml"
    # containerd 配置文件
    etcContainerd="/etc/containerd/config.toml"
    if [ ! -d "/etc/containerd" ]; then mkdir -p "/etc/containerd"; fi
    # containerd 证书保存目录
    certsDirContainerd="/etc/containerd/certs.d"
    if [ ! -d "${certsDirContainerd}" ]; then mkdir -p "${certsDirContainerd}"; fi
    # containerd 服务文件
    srvFileContainerd="/etc/systemd/system/containerd.service"
    # containerd 数据存储目录
    if [ "${containerdDataDir}" != "" ]; then mkdir -p "${containerdDataDir}"; fi

    # docker daemon 配置文件
    dockerDaemonFile="/etc/docker/daemon.json"
    # docker 数据目录
    dockerDataDir="/data/docker"
    if [ ! -d "${dockerDataDir}" ]; then mkdir -p "${dockerDataDir}"; fi
    # docker 服务文件 预留
    # srvFileDocker="/etc/systemd/system/docker.service"
    # docker 的 socket 配置文件 预留
    # etcSocketDocker="/etc/systemd/system/docker.socket"
    # docker 保存证书的目录
    certsDirDocker="/etc/docker/certs.d"
    if [ ! -d "${certsDirDocker}" ]; then mkdir -p "${certsDirDocker}"; fi
    # cri-docker 的 socket 配置文件 预留
    etcCriDockerdSocket="/etc/systemd/system/cri-docker.socket"
    # cri-docker 的 服务管理文件 预留
    srvFileCriDockerd="/etc/systemd/system/cri-docker.service"

    # keepalived 配置文件
    etcFileKeepalived="/etc/keepalived/keepalived.conf"
    # haproxy 配置文件
    etcFileHaproxy="/etc/haproxy/haproxy.cfg"

    # kubeadm conf 配置文件
    etcKubeadmConf="/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"

    # kubelet 服务控制文件
    srvFileKubelet="/etc/systemd/system/kubelet.service"
    # kubelet 的数据和日志存储目录
    if [ "${kubeletDataDir}" != "" ]; then mkdir -p "${kubeletDataDir}"/data; fi
    # etcd 的数据存储目录
    if [ "${etcdDataDir}" != "" ]; then mkdir -p "${etcdDataDir}"/data; fi

    # k8s 集群初始化配置文件 - 脚本自动创建
    initFile="${sDir}/kubeadm-init.yaml"
    # k8s 集群所需离线镜像列表文件 - 脚本自动创建
    # imgListFile="${sDir}/k8s-images.list"

    # calico network 配置文件
    calicoNetConf="/etc/NetworkManager/conf.d/calico.conf"

    # 当 imagePolicy="2" 即使用官方离线镜像时 脚本将强制修改为官方地址
    if [ "${imagePolicy}" = "2" ]; then
        imgRegAddr="registry.k8s.io"
        calicoRegAddr="docker.io/calico"
        metricsServerRegAddr="registry.k8s.io/metrics-server"
    fi

    # cri-runtime sock
    case "${criRuntime}" in
        "containerd" ) criSockFile="${containerdSock}"; ;;
        "docker" ) criSockFile="${criDockerdSock}"; ;;
        "podman" ) criSockFile="${criDockerdSock}"; ;;
    esac
}

# 脚本参数校验
function chk_params() {
    oput red "脚本参数校验"
    echo
    case "${criRuntime}" in
        "containerd" | "docker" )
            echo "$(oput blue "criRuntime") 校验 $(oput green "已通过")"
            ;;
        * )
            echo
            echo "$(oput blue "criRuntime") 校验 $(oput red "未通过") 只能是 $(oput blue "containerd|docker")"
            return 1
            ;;
    esac

    # master 节点数量
    mSum=${#k8sNodesMaster[@]}
    # worker 节点数量
    wSum=${#k8sNodesWorker[@]}
    if [ "${mSum}" -ge 1 ]; then
        # master 节点去掉第一个节点后的数量
        # mSumNo1st=$((mSum-1))
        # 检查worker节点数量是否需要按照奇数递增
        if ((mSum % 2 == 0)); then
            echo "$(oput blue "k8sNodesMaster") 校验 $(oput green "未通过") 要求至少 1 个 master 节点 并且需按照奇数递增)"
            return 1
        else
            echo "$(oput blue "k8sNodesMaster") 校验 $(oput green "已通过")"
        fi
    else
        echo "$(oput blue "k8sNodesMaster") 校验 $(oput green "未通过") 要求至少 1 个 master 节点 并且需按照奇数递增)"
        return 1
    fi
    if [ "${wSum}" -ge 1 ]; then
        echo "$(oput blue "k8sNodesWorker") 校验 $(oput green "已通过")"
    else
        echo "$(oput blue "k8sNodesWorker") 校验 $(oput green "未通过") 要求至少 1 个 worker 节点 节点递增无特殊要求)"
        return 1
    fi

    case "${imagePolicy}" in
        "1" | "2" )
            echo "$(oput blue "imagePolicy") 校验 $(oput green "已通过")"
            ;;
        * )
            oput red "$(oput blue "imagePolicy") 校验 $(oput red "未通过") 只能是 $(oput blue "1") 或 $(oput blue "2")"
            return 1
            ;;
    esac

    case "${k8sVer%.*}" in
        1.[2-9][4-9] | 1.[3][0-9] )
            echo "$(oput blue "k8sVer") 校验 $(oput green "已通过")"
            ;;
        * )
            oput red "$(oput blue "k8sVer") 校验 $(oput red "未通过") 只能是 $(oput blue "1.24.0+") 例如: $(oput blue "1.24.0/1.28.5")"
            return 1
            ;;
    esac

    if ! chk_dup_ip_hostname; then return 1; fi
}

# 检查所有节点是否存在重复IP和主机名
function chk_dup_ip_hostname() {
    # 初始化变量
    declare -A ipMap
    declare -A nameMap

    # 读取文件并校验
    while IFS= read -r line; do
        # 解构IP和节点名
        IFS=' ' read -r ip name <<< "$line"

        # 检查IP是否重复
        if [[ -n "${ipMap[$ip]}" ]]; then
            echo
            echo "参数错误: 节点 $(oput red "${ipMap[$ip]}") 和 $(oput red "${name}") 的 IP 地址 $(oput red "${ip}") 重复"
            return 1
        fi
        ipMap[$ip]=$name

        # 检查节点名是否重复
        if [[ -n "${nameMap[$name]}" ]]; then
            echo
            echo "参数错误: IP $(oput red "${nameMap[$name]}") 和 $(oput red "${ip}") 的节点名 $(oput red "${name}") 重复"
            return 1
        fi
        nameMap[$name]=$ip
    done < "${k8sNodesListFile}"
}

# 检查SSH远程主机连通性
function chk_ssh_conn() {
    # 需要指定绝对路径的 ip.list 否则默认在 gossh 程序所在的目录进行查找
    # bin/amd64/gossh -t cmd -i /seeyon/scripts/ip.list -P 22 -u root -p 110110 -f "hostname"
    oput red "检查远程节点SSH连通性"
    echo
    local ip
    local id=0; local ok=0; local no=0
    for x in "${k8sNodesAll[@]:1}"; do
        id=$((id+1))
        ip="$(echo "${x}" | cut -d ' ' -f1)"
        echo -e "${id}/$((${#k8sNodesAll[@]}-1)) 正在检查: $(oput blue "${ip}") \c"
        CMD="gossh -t cmd -h ${ip} ${gOpts} \"hostname\""
        if eval "${CMD}" | grep -q 'return=0'; then
            ok=$((ok+1))
            oput green "已通过"
        else
            no=$((no+1))
            oput red "未通过"
        fi
    done
    if [ "${no}" -eq 0 ]; then
        echo; oput green "所有节点SSH连通性检查通过"
    else
        echo; oput red "有节点SSH连通性检查未通过"
        return 1
    fi
}

# 检查必需的安装文件和模板文件是否存在
function find_files() {
    oput red "查找必需文件"; echo
    # find 命令选项
    findOpts="-maxdepth 1 -regextype \"posix-extended\" -regex"
    if sort --help | grep -Ewq '\-V,'; then
        fileSort="sed 's/^\.\///' | sort -V | tail -1"
    else
        fileSort="sed 's/^\.\///' | sort | tail -1"
    fi

    fileList=(
        "cni-plugins"
        "crictl"
        "kubeadm"
        "kubectl"
        "kubelet"
        "runc"
    )
    case "${criRuntime}" in
        "containerd" )
            fileList+=("containerd")
            ;;
        "docker" )
            # fileList+=("docker" "cri-docker" "docker-compose" "docker-buildx")
            fileList+=("cri-docker")
            ;;
    esac
    case "${imagePolicy}" in
        "1" )
            # fileList+=()
            ;;
        "2" )
            fileList+=(
                "coredns"
                "etcd"
                "kube-apiserver"
                "kube-controller-manager"
                "kube-proxy"
                "kube-scheduler"
                "pause"
                "calico-cni"
                "calico-node"
                "calico-kube-controllers"
            )
            ;;
    esac
    if [ "${installHelm}" = "1" ]; then fileList+=("helm"); fi
    if [ "${installMetricsServer}" = "1" ]; then fileList+=("metrics-server"); fi

    # 如果需要限制版本号则在此处调整正则条件
    for x in "${fileList[@]}"; do
        case "${x}" in
            "cni-plugins" )
                findRegex="cni-plugins-linux-${archType}-v1.[3-9]+.[0-9]+.tgz"
                webSite="https://github.com/containernetworking/plugins/releases"
                ;;
            "containerd" )
                findRegex="containerd-1.[6-9].[0-9]+-linux-${archType}.tar.gz"
                webSite="https://github.com/containerd/containerd/releases"
                ;;
            "crictl" )
                findRegex="crictl-v1.([2][4-9]|[3-9][0-9]).[0-9]-linux-${archType}.tar.gz"
                webSite="https://github.com/kubernetes-sigs/cri-tools/releases"
                ;;
            "kubeadm" | "kubectl" | "kubelet" )
                findRegex="${x}"
                webSite="https://github.com/kubernetes/kubernetes/releases"
                ;;
            "runc" )
                findRegex="runc.${archType}"
                webSite="https://github.com/opencontainers/runc/releases"
                ;;
            # "docker" )
            #     findRegex="docker-2[0-9].[0-9]+.[0-9]+.tgz"
            #     webSite="https://download.docker.com/linux/static/stable/$(arch)"
            #     ;;
            "cri-docker" )
                 findRegex="cri-dockerd-[0-9].[0-9]+.[0-9]+.${archType}.tgz"
                 webSite="https://github.com/Mirantis/cri-dockerd/releases"
                 ;;
            # "docker-compose" )
            #     findRegex="docker-compose-linux-$(arch)"
            #     webSite="https://github.com/docker/compose/releases"
            #     ;;
            # "docker-buildx" )
            #     findRegex="buildx-v[0-9].[0-9]+.[0-9]+.linux-${archType}"
            #     webSite="https://github.com/docker/buildx/releases"
            #     ;;
            "coredns" )
                findRegex="coredns.*.tar"
                webSite="https://github.com/coredns/coredns/releases"
                ;;
            "etcd" )
                findRegex="etcd.*.tar"
                webSite="https://quay.io/repository/coreos/etcd"
                ;;
            "kube-apiserver" )
                findRegex="kube-apiserver.*.tar"
                webSite="https://github.com/kubernetes/kubernetes/releases"
                ;;
            "kube-controller-manager" )
                findRegex="kube-controller-manager.*.tar"
                webSite="https://github.com/kubernetes/kubernetes/releases"
                ;;
            "kube-proxy" )
                findRegex="kube-proxy.*.tar"
                webSite="https://github.com/kubernetes/kubernetes/releases"
                ;;
            "kube-scheduler" )
                findRegex="kube-scheduler.*.tar"
                webSite="https://github.com/kubernetes/kubernetes/releases"
                ;;
            "pause" )
                findRegex="pause.*.tar"
                webSite="请自行百度"
                ;;
            "calico-cni" )
                findRegex="calico-cni.*.tar"
                webSite="https://github.com/projectcalico/calico/releases"
                ;;
            "calico-node" )
                findRegex="calico-node.*.tar"
                webSite="https://github.com/projectcalico/calico/releases"
                ;;
            "calico-kube-controllers" )
                findRegex="calico-kube-controllers.*.tar"
                webSite="https://github.com/projectcalico/calico/releases"
                ;;
            "helm" )
                findRegex="helm-v3.[0-9.]+-linux-${archType}.tar.gz"
                webSite="https://helm.sh"
                ;;
            "metrics-server" )
                findRegex="metrics-server-v0.[0-9.]+.*.tar"
                webSite="https://github.com/kubernetes-sigs/metrics-server"
                ;;
            "metrics-scraper" )
                findRegex="metrics-scraper-v1.[0-9.]+.*.tar"
                webSite="https://github.com/kubernetes/dashboard"
                ;;
            * )
                echo; oput red "app not found: ${x}"; return 1;
                ;;
        esac

        case "${imagePolicy}" in
            "2" )
                case "${x}" in
                    "coredns" | "etcd" | "kube-apiserver" \
                        | "kube-controller-manager" | "kube-proxy" | "kube-scheduler" | "pause" \
                        | "calico-cni" | "calico-node" | "calico-kube-controllers" \
                        | "metrics-server" | "metrics-scraper" )
                    msgName="镜像文件"
                        ;;
                    * )
                        msgName="安装文件"
                        ;;
                esac
                ;;
            * )
                msgName="安装文件"
                ;;
        esac
        if [ "${imagePolicy}" = "2" ]; then
            findCMD="find ${softwareDir} ${imgDir} ${findOpts} \".*${findRegex}\" -type f | ${fileSort}"
        else
            findCMD="find ${softwareDir} ${findOpts} \".*${findRegex}\" -type f | ${fileSort}"
        fi
        if appInstallFile="$(eval "${findCMD}" 2>>"${sLog}")"; then
            if [ "${appInstallFile}" != "" ]; then
                appInstallFileSize="$(du -sh "${appInstallFile}" | awk '{print $1}')"
                echo "$(oput green "已找到") $(oput blue "${x}") 的${msgName} $(oput blue "${appInstallFile}") $(oput green "${appInstallFileSize}")"
                case "${x}" in
                    "cni-plugins" )
                        cniPluginsFile="${appInstallFile}"
                        ;;
                    "containerd" )
                        containerdFile="${appInstallFile}"
                        ;;
                    "crictl" )
                        crictlFile="${appInstallFile}"
                        ;;
                    "kubeadm" )
                        kubeadmFile="${appInstallFile}"
                        ;;
                    "kubectl" )
                        kubectlFile="${appInstallFile}"
                        ;;
                    "kubelet" )
                        kubeletFile="${appInstallFile}"
                        ;;
                    "runc" )
                        runcFile="${appInstallFile}"
                        ;;
                    # "docker" )
                    #    dockerFile="${appInstallFile}"
                    #    ;;
                    "cri-docker" )
                        criDockerFile="${appInstallFile}"
                        ;;
                    # "docker-compose" )
                    #    dockerComposeFile="${appInstallFile}"
                    #    ;;
                    # "docker-buildx" )
                    #    dockerBuildxFile="${appInstallFile}"
                    #    ;;
                    "coredns" )
                        corednsFile="${appInstallFile}"
                        ;;
                    "etcd" )
                        etcdFile="${appInstallFile}"
                        ;;
                    "kube-apiserver" )
                        kubeApiFile="${appInstallFile}"
                        ;;
                    "kube-controller-manager" )
                        kubeCMFile="${appInstallFile}"
                        ;;
                    "kube-proxy" )
                        kubeProxyFile="${appInstallFile}"
                        ;;
                    "kube-scheduler" )
                        kubeSchFile="${appInstallFile}"
                        ;;
                    "pause" )
                        pauseFile="${appInstallFile}"
                        ;;
                    "calico-cni" )
                        calicoCniFile="${appInstallFile}"
                        ;;
                    "calico-node" )
                        calicoNodeFile="${appInstallFile}"
                        ;;
                    "calico-kube-controllers" )
                        calicoKCFile="${appInstallFile}"
                        ;;
                    "helm" )
                        helmFile="${appInstallFile}"
                        ;;
                    "metrics-server" )
                        metricsServerFile="${appInstallFile}"
                        ;;
                    "metrics-scraper" )
                        metricsScraperFile="${appInstallFile}"
                        ;;
                    * )
                        echo; oput red "app not found: ${x}"; return 1;
                        ;;
                esac
            else
                echo "$(oput red "未找到") $(oput blue "${x}") 的${msgName} 下载地址 $(oput blue "${webSite}")"
                return 1
            fi
        else
            oput red "查找命令: 执行失败 ${findCMD}"
            return 1
        fi
    done
    # 检查必需的模板文件是否存在
    case "${hostAddr}" in
        "$(echo "${k8sNodesMaster[0]}" | awk '{print $1}')" )
            templateFileCalico="${templateDir}/calico-${calicoVer}.yaml"
            templateFilemetricsServer="${templateDir}/metrics-server.yaml"

            templateFileList=("${templateFileCalico}")
            if [ "${installMetricsServer}" = 1 ]; then
                templateFileList+=("${templateFilemetricsServer}")
            fi

            local msgAppName
            for templateFile in "${templateFileList[@]}"; do
                if echo "${templateFile}" | grep -q calico; then
                    msgAppName="calico"
                elif echo "${templateFile}" | grep -q metrics-server; then
                    msgAppName="metrics-server"
                fi

                if [ -f "${templateFile}" ]; then
                    echo "$(oput green "已找到") $(oput blue "${msgAppName}") 的模板文件 $(oput blue "${templateFile}")"
                else
                    echo "$(oput red "未找到") $(oput blue "${msgAppName}") 的模板文件 $(oput blue "${templateFile}")"
                    return 1
                fi
            done
            ;;
    esac
    # 当进行安装时 获取 pause 的版本号
    if [[ "${action}" == "i" || "${action}" == "install" ]]; then get_pause_ver; fi
}

function try_unpack_pkg() {
    # criRuntime 一般情况下大小版本匹配k8s的版本 修订版本一般固定为0或1 绝大多数情况下是0
    # 当版本是1.26时最新的是 1.26.1 如果版本不匹配需手动修改 例如: "k8s-criRuntime-v1.26.1-${archType}.7z"
    local pkgList
    local pkgList=(
        "calico-images-v${calicoVer}-${archType}.7z"
        "k8s-binaries-v${k8sVer}-${archType}.7z"
        # "k8s-criRuntime-v${k8sVer%.*}.1-${archType}.7z"
        "k8s-images-v${k8sVer}-${archType}.7z"
    )
    if [ "${installMetricsServer}" = "1" ]; then
        pkgList+=("metrics-server-images-v${metricsServerVer}-${archType}.7z")
    fi
    if [ -f "k8s-criRuntime-v${k8sVer%.*}.1-${archType}.7z" ]; then
        pkgList+=("k8s-criRuntime-v${k8sVer%.*}.1-${archType}.7z")
    else
        pkgList+=("k8s-criRuntime-v${k8sVer%.*}.0-${archType}.7z")
    fi

    for pkg in "${pkgList[@]}"; do
        echo -e "查找文件: $(oput blue "${pkg}") \c"
        if [ -f "${pkg}" ]; then
            oput green "已找到"
            okList+=("${pkg}")
        else
            oput red "未找到"
            noList+=("${pkg}")
        fi
    done

    if [ "${#okList[@]}" -eq "${#pkgList[@]}" ]; then
        echo
        read -rp "$(oput green "检测到离线部署 k8s 集群相关的所有的 7z 格式压缩包文件 是否自动解压? (y/N)") " yn
        echo
        case "${yn}" in
            [yY] )
                id=0
                for pkg in "${pkgList[@]}"; do
                    id=$((id+1))
                    echo -e "${id}/${#pkgList[@]} 正在解压: $(oput blue "${pkg}") \c"
                    if 7zzs x -y "${pkg}" &>>"${sLog}"; then
                        oput green "成功"
                    else
                        oput red "失败"
                        return 1
                    fi
                done
                ;;
            * )
                return 0
                ;;
        esac
    fi
}

# 获取 pause 的版本号
function get_pause_ver() {
    echo; echo -e "正获取 $(oput blue "pause") 版本号: \c"
    if [ ! -f  "${kubeadmFile}" ]; then oput red "失败: 文件不存在 ${kubeadmFile}"; return 1; fi
    chmod +x "${kubeadmFile}" &>/dev/null
    pauseVer="$(eval "${kubeadmFile}" config images list 2>/dev/null | grep 'pause' | awk -F ':' '{print $(NF)}')"
    if [ -z "${pauseVer}" ]; then
        oput red "失败"; return 1
    else
        echo "$(oput green "成功") 版本号是 $(oput blue "${pauseVer}")"
    fi
}

# 指定 IP 地址获取其关联的网卡名称
function get_eth_name() {
    local ipAddr="$1"
    ethName="$(ip addr show | grep "${ipAddr}" | awk '{print $(NF)}')"
    if [ -z "${ethName}" ]; then return 1; fi
}

# 调整系统参数
function tune_system() {
    oput red "调整系统参数"

    # 设置时区
    echo; echo -e "设置系统时区: $(oput blue "${osTimeZone}") \c"
    if timedatectl set-timezone "${osTimeZone}" >>"${sLog}"; then oput green "成功"; else oput red "失败"; return 1; fi

    # 设置主机名
    for pair in "${k8sNodesAll[@]}"; do
        local ip
        local hostName
        ip="$(echo "${pair}" | awk '{print $1}')"
        hostName=$(echo "${pair}" | awk '{print $2}')
        if [ "${hostAddr}" = "${ip}" ]; then
            echo; echo -e "设置主 机 名: $(oput blue "${hostName}") \c"
            if hostnamectl set-hostname "${hostName}" >>"${sLog}"; then oput green "成功"; else oput red "失败"; return 1; fi
            break
        fi
    done

    # 设置交换分区
    echo -e "禁用交换分区: \c"
    swapoff -a
    sed -i '/^[^#].*swap/ s/^/# /' /etc/fstab
    oput green "完成"

    # 设置防火墙
    echo -e "禁用OS防火墙: \c"
    case "${osFamily}" in
        "redhat" )
            systemctl stop firewalld &>/dev/null
            if systemctl disable firewalld &>/dev/null; then oput green "成功"; else oput red "失败"; fi
            ;;
        "debian" )
            oput yellow "debian 家族的操作系统请手动关闭防火墙"
            ;;
    esac

    # 设置selinux
    echo -e "禁用 selinux: \c"
    if [ -s /etc/selinux/config ]; then
        # echo; echo "配置文件: /etc/selinux/config"
        if grep -q 'SELINUX=enforcing' /etc/selinux/config; then
            sed -i 's/SELINUX=enforcing/SELINUX=disabled/g' /etc/selinux/config
            # echo "已经调整配置: SELINUX=disabled"
        else
            :
            # echo "无需再次配置: SELINUX=disabled"
        fi
    fi
    setenforce 0 &>/dev/null
    oput green "完成"

    # 禁止来自 systemd-journald 的广播消息打印在控制台影响脚本美观
    # 如下屏幕输出:
    # Broadcast message from systemd-journald@k8s-m1 (Sat 2023-12-23 12:42:45 CST):
    # haproxy[3961]: backend kube-apiserver has no server available!
    if [ -f "/etc/systemd/journald.conf" ]; then
        echo -e "禁止日志广播: \c"
        sed -i 's?^#ForwardToWall=yes?ForwardToWall=no?' /etc/systemd/journald.conf
        systemctl daemon-reload; systemctl restart systemd-journald >/dev/null 2>&1
        oput green "完成"
    fi

    # 设置系统内核
    hostsFile="/etc/hosts"
    limitsFile="/etc/security/limits.conf"
    sysctlFile="/etc/sysctl.conf"
    ipvsFile="/etc/modules-load.d/ipvs.conf"
    overlayBrFile="/etc/modules-load.d/overlay-br.conf"

    local destFileList
    destFileList=(
        "${hostsFile}"
        "${limitsFile}"
        "${sysctlFile}"
        "${ipvsFile}"
        "${overlayBrFile}"
        )

    local id varList
    id=0
    local f
    for f in "${destFileList[@]}"; do
        id=$((id+1))
        case "${f}" in
            "/etc/hosts" )
                varList=(
                    "# set k8s cluster hostname mapping"
                    "${vipAddr} ${vipHostName} ${vipDomain}"
                )
                for x in "${k8sNodesMaster[@]}"; do varList+=("${x}"); done
                for x in "${k8sNodesWorker[@]}"; do varList+=("${x}"); done
                ;;
            "/etc/sysctl.conf" )
                varList=(
                    "fs.file-max=52706963"
                    "fs.inotify.max_user_watches=89100"
                    "fs.may_detach_mounts=1"
                    "fs.nr_open=52706963"
                    "kernel.panic=3"
                    "kernel.sysrq=0"
                    "net.bridge.bridge-nf-call-ip6tables=1"
                    "net.bridge.bridge-nf-call-iptables=1"
                    "net.core.busy_read=100"
                    "net.core.somaxconn=16384"
                    "net.ipv4.conf.all.accept_source_route=0"
                    "net.ipv4.conf.all.promote_secondaries=1"
                    "net.ipv4.ip_conntrack_max=65536"
                    "net.ipv4.ip_forward=1"
                    "net.ipv4.tcp_keepalive_intvl=15"
                    "net.ipv4.tcp_keepalive_probes=3"
                    "net.ipv4.tcp_keepalive_time=600"
                    "net.ipv4.tcp_max_orphans=327680"
                    "net.ipv4.tcp_max_syn_backlog=16384"
                    "net.ipv4.tcp_max_tw_buckets=36000"
                    "net.ipv4.tcp_orphan_retries=3"
                    "net.ipv4.tcp_syncookies=1"
                    "net.ipv4.tcp_timestamps=0"
                    "net.ipv4.tcp_tw_reuse=1"
                    "net.ipv6.conf.all.disable_ipv6=0"
                    "net.ipv6.conf.all.forwarding=1"
                    "net.ipv6.conf.default.disable_ipv6=0"
                    "net.ipv6.conf.lo.disable_ipv6=0"
                    "net.netfilter.nf_conntrack_max=2310720"
                    "vm.dirty_ratio=50"
                    "vm.overcommit_memory=1"
                    "vm.panic_on_oom=0"
                )
                ;;
            "/etc/security/limits.conf" )
                varList=(
                    "* soft nofile 655360"
                    "* hard nofile 131072"
                    "* soft nproc 655350"
                    "* hard nproc 655350"
                    "* soft memlock unlimited"
                    "* hard memlock unlimited"
                )
                ;;
            "/etc/modules-load.d/ipvs.conf" )
                touch "/etc/modules-load.d/ipvs.conf"
                varList=(
                    "ipip"
                    "ip_set"
                    "ip_tables"
                    "ipt_REJECT"
                    "ipt_rpfilter"
                    "ipt_set"
                    "ip_vs"
                    "ip_vs_dh"
                    "ip_vs_ftp"
                    "ip_vs_lblc"
                    "ip_vs_lblcr"
                    "ip_vs_lc"
                    "ip_vs_nq"
                    "ip_vs_rr"
                    "ip_vs_sed"
                    "ip_vs_sh"
                    "ip_vs_wlc"
                    "ip_vs_wrr"
                    "nf_conntrack"
                    "xt_set"
                )
                ;;
            "/etc/modules-load.d/overlay-br.conf" )
                touch "/etc/modules-load.d/overlay-br.conf"
                varList=(
                    "overlay"
                    "br_netfilter"
                )
                ;;
        esac

        echo; echo "$(oput green "${id}")/$(oput red "${#destFileList[*]}") 配置文件: $(oput blue "${f}")"
        if [ ! -f "${f}" ]; then oput red "文件不存在: ${f}"; continue; fi
        local var
        for var in "${varList[@]}"; do
            if grep -q "^${var}" "${f}"; then
                echo "无需再次添加: ${var}"
            else
                if echo "${var}" >> "${f}" 2>>"${sLog}"; then
                    echo "添加参数成功: ${var}"
                else
                    echo "添加参数失败: ${var}"
                fi
            fi
            varList=()
        done
    done

    {
        systemctl restart systemd-modules-load.service
        sysctl --system
        modprobe overlay
        modprobe br_netfilter
    } >> "${sLog}" 2>&1

    {
        # 确认系统变量在 sysctl 配置中是否被设置为 1 有时候需要重启服务器后再做一次验证检查
        sysctl net.bridge.bridge-nf-call-iptables net.bridge.bridge-nf-call-ip6tables net.ipv4.ip_forward
        # 确认 br_netfilter 和 overlay 模块被加载
        lsmod | grep br_netfilter; lsmod | grep overlay
        # 查看ipvs模式是否启用成功
        ipvsadm -Ln
    } >> "${sLog}" 2>&1
}

# 检查脚本所需命令是否存在
function chk_command() {
    oput red "检查必需命令"; echo
    local cmdList
    local cmd
    local count
    local cmdVer
    missCount=0
    # cmdList=(gossh sshpass yq usql)
    # cmdList=(gossh sshpass)
    cmdList=(gossh)
    for cmd in "${cmdList[@]}"; do
        # echo -e "正在查找命令: \c"
        if command -v "${cmd}" &>/dev/null; then
            case "${cmd}" in
                "gossh" )
                    cmdVer="$(gossh --version | awk '{print $2}')"
                    ;;
                "sshpass" )
                    cmdVer="$(sshpass -V | grep -E '^sshpass' | awk '{print $2}')"
                    ;;
                "yq" )
                    cmdVer="$(yq -V | grep -Ewo 'v[0-9.]+' | awk -F 'v' '{print $2}')"
                    ;;
                "usql" )
                    cmdVer="$(usql --version | awk '{print $2}')"
                    ;;
            esac
            echo "$(oput green "已找到") $(oput blue "${cmd}") $(oput green "${cmdVer}")"
        else
            echo "$(oput red "未找到") $(oput blue "${cmd}")"
            missCount=$((missCount+1))
        fi
    done
    if [ "${missCount}" -gt 0 ]; then
        echo; echo "缺少 $(oput red "${missCount}") 个必需命令 请检查 $(oput blue "${sBinDir}") 下是否存在这些命令文件"
        return 1
    fi
}

# 检查脚本所需依赖
function chk_depend() {
    oput red "检查系统依赖"; echo
    local dependList
    dependList=(
        conntrack
        curl
        ebtables
        ethtool
        ipset
        ipvsadm
        iproute-tc
        psmisc
        socat
        sshpass
        sysstat
        tar
        wget
        telnet
        vim
        zip
    )
#    case "${criRuntime}" in
#        "docker" )
#            dependList+=(iptables libcgroup libcgroup-tools fuse-overlayfs fuse3 fuse3-libs)
#            ;;
#    esac

    local id=0
    local count=0
    local oneDepend
    local dependCheckCMD dependInstallCMD
    for oneDepend in "${dependList[@]}"; do
        id="$((id+1))"
        case "${osFamily}" in
            "redhat" )
                dependCheckCMD="${osPCMD} -q --queryformat %-{name}\\n ${oneDepend}"
                dependInstallCMD="${osDCMD} install -y ${oneDepend}"
                ;;
            "debian" )
                dependCheckCMD="${osPCMD} -l ${oneDepend} | grep -qwo '^ii.*${oneDepend}'"
                dependInstallCMD="${osDCMD} install -y ${oneDepend}"
                ;;
        esac
        if eval "${dependCheckCMD}" &>/dev/null; then
            echo -e "${id}/${#dependList[@]} 无需安装 ${oneDepend}"
        else
            if [ "${installDepend}" = "1" ]; then
                echo -e "${id}/${#dependList[@]} 正在安装 ${oneDepend} \c"
                if eval "${dependInstallCMD}" &>>"${sLog}"; then
                    oput green "成功"
                else
                    if [ "${oneDepend}" = "iproute-tc" ]; then
                        oput red "失败 ${dependInstallCMD} 部分操作系统无此包时可忽略"
                    else
                        oput red "失败 ${dependInstallCMD}"
                    fi
                fi
            else
                echo -e "${id}/${#dependList[@]} 需要安装 ${oneDepend} 参考命令: ${dependInstallCMD}"
                count="$((count+1))"
            fi
        fi
    done

    if [ "${count}" -gt 0 ]; then
        oput red "部署 k8s 高可用集群之前 还有 ${count} 个依赖需要安装"
        return 1
    else
        return 0
    fi
}

# 获取所有节点 ip 和 主机名 为一个全局数组 并且保存到文件
function fetch_nodes() {
    # oput red "保存节点信息"; echo
    k8sNodesListFile="${sDir}/k8sNodesAll.list"
    touch "${k8sNodesListFile}" && true >"${k8sNodesListFile}"
    k8sNodesAll=()
    for x in "${k8sNodesMaster[@]}"; do
        k8sNodesAll+=("${x}")
        echo "${x}" >> "${k8sNodesListFile}"
    done
    for x in "${k8sNodesWorker[@]}"; do
        k8sNodesAll+=("${x}")
        echo "${x}" >> "${k8sNodesListFile}"
    done
    if [ ! -s "${k8sNodesListFile}" ]; then
        # echo "集群节点信息: $(oput red "保存到文件失败")"
        return 1
    else
        # echo "集群节点信息: $(oput green "保存到文件成功 ${k8sNodesListFile}")"
        return 0
    fi
}

# 检查并设置免密登录
function set_nopass() {
    oput red "检查免密登录"; echo
    # 检查是否已经设置过免密登录 如果没有设置过就设置免密登录
    if [ -f "/root/.ssh/id_rsa" ]; then
        oput green "无需再次设置"
        return 0
    fi
    ssh-keygen -f "/root/.ssh/id_rsa" -P '' &>> "${sLog}" 2>&1
    # ssh-keygen -f "/root/.ssh/id_rsa" -P '' -y &>> "${sLog}"
    export SSHPASS=${sshPWD}
    local nodeIP
    for host in "${k8sNodesAll[@]}"; do
        nodeIP="$(echo "${host}" | awk '{print $1}')"
        if sshpass -e ssh-copy-id -o StrictHostKeyChecking=no "${nodeIP}" &>> "${sLog}"; then
            echo "设置免密登录: $(oput blue "${nodeIP}") $(oput green "成功")"
        else
            echo "设置免密登录: $(oput blue "${nodeIP}") $(oput red "失败")"
            # return 1
        fi
    done
}

# 检查软件是否已安装 已装返回 0 未装返回 1 接受字符串类型的数组
function chk_install() {
    echo; oput red "检查已装软件"; echo
    local -a appList=("$@")
    local app
    local missingCount
    missingCount=0
    local id
    id=0
    # read -ra appList <<< "$1"
    for app in "${appList[@]}"; do
        id=$((id+1))
        case "${app}" in
            "host-local" )
                echo -e "${id}/${#appList[@]} 软件 $(oput blue "cni-plugins") \c"
                ;;
            "crictl" )
                echo -e "${id}/${#appList[@]} 软件 $(oput blue "cri-tools") \c"
                ;;
            * )
                echo -e "${id}/${#appList[@]} 软件 $(oput blue "${app}") \c"
                ;;
        esac
        if command -v "${app}" >/dev/null 2>&1; then
            oput green "已安装"
        else
            oput red "未安装"
            missingCount=$((missingCount+1))
        fi
    done
    if [ "${missingCount}" -gt 0 ]; then return 1; else return 0; fi
}

# 检查服务是否正在运行并根据脚本参数设定是否自动关闭服务 接受数组参数
function chk_srv() {
    echo; oput red "检查服务状态"; echo
    # 注意 local -a srvList=("$@") 和 chk_install 函数中的 read -ra appList <<< "$1" 这两者的用法
    local id=0
    local srvRuningCount=0
    local -a srvList=("$@")
    # read -ra srvList <<< "$1"
    for srv in "${srvList[@]}"; do
        id=$((id+1))
        if systemctl is-active --quiet "${srv}"; then
            echo "${id}/${#srvList[@]} 服务 $(oput blue "${srv}") $(oput red "正在运行")"
            srvRuningCount=$((srvRuningCount+1))
            if [ "${autoStopSrv}" = "1" ]; then
                echo -e "正在关闭 $(oput blue "${srv}") 服务 \c"
                if systemctl stop "${srv}" &>>"${sLog}"; then
                    oput green "成功"
                    srvRuningCount=$((srvRuningCount-1))
                else
                    oput red "失败"
                fi
            else
                echo "执行 $(oput blue "systemctl stop ${srv}") $(oput red "手动关闭") $(oput blue "${srv}") 服务后再运行脚本"
                # 如果要提前退出函数，不执行后续的计数比较 则取消下行的 return 1 的注释
                # return 1
            fi
        else
            echo "${id}/${#srvList[@]} 服务 $(oput blue "${srv}") $(oput green "没有运行")"
        fi
    done
    if [ "${srvRuningCount}" -gt 0 ]; then
        return 0  # 返回0表示有服务正在运行，需要关闭
    else
        return 1  # 返回1表示所有服务都已关闭，无需操作
    fi
}

# 设置开机自启和启动服务 接受数组参数
function set_bootup() {
    echo; oput red "设置开机自启和启动服务"; echo
    local -a srvList=("${@}")
    # read -ra srvList <<< "$1"
    for srv in "${srvList[@]}"; do
        echo -e "设置开机自启: $(oput blue "${srv}") \c"
        if systemctl enable "${srv}" >/dev/null 2>&1; then oput green "成功"; else oput red "失败"; return 1; fi
        echo -e "正在启动服务: $(oput blue "${srv}") \c"
        if systemctl restart "${srv}" >/dev/null 2>&1; then oput green "成功"; else oput red "失败"; return 1; fi
    done
}

# 安装 cri-tools
function install_crictl() {
    echo -e "安装软件: $(oput blue "cri-tools") \c"
    if tar -xf "${crictlFile}" -C "${installRootDir}/bin"; then
        oput green "成功"
        echo -e "创建文件: $(oput blue "${etcYamlCrictl}") \c"
        cat > "${etcYamlCrictl}" <<EOF
runtime-endpoint: ${criSockFile}
image-endpoint: ${criSockFile}
timeout: 10
debug: false
EOF
        if [ -s "${etcYamlCrictl}" ]; then oput green "成功"; else oput red "失败"; return 1; fi
    else
        oput red "失败"; return 1
    fi
}

# 安装 containerd 容器运行时
function install_containerd() {
    oput red "安装 containerd 容器运行时"

    appList=("containerd"); srvList=("containerd")
    if chk_install "${appList[@]}"; then
        case "${actOnExist}" in
            "1" )
                echo; echo "根据脚本参数设定 即将 $(oput red "覆盖安装") $(oput blue "${appList[*]}")"
                ;;
            "2" )
                echo; echo "根据脚本参数设定 即将 $(oput yellow "跳过安装") $(oput blue "${appList[*]}")"
                return 0
                ;;
            * )
                echo; echo "根据脚本参数设定 即将 $(oput blue "全新安装") $(oput blue "${appList[*]}")"
                ;;
        esac
    fi
    if chk_srv "${srvList[@]}"; then return 1; fi

    # containerd
    echo; echo -e "安装软件: $(oput blue "containerd") \c"
    if tar -xf "${containerdFile}" -C "${installRootDir}"; then
        oput green "成功"
    else
        oput red "失败"; return 1
    fi
    echo -e "创建文件: $(oput blue "${etcContainerd}") \c"
    if containerd config default > "${etcContainerd}"; then
        oput green "成功"
    else
        oput red "失败"; return 1
    fi
    echo "调整配置: $(oput blue "sandbox_image = ${imgRegAddr}/pause:${pauseVer}")"
    sed -i "s%sandbox_image = .*%sandbox_image = \"${imgRegAddr}/pause:${pauseVer}\"%" "${etcContainerd}"
    echo "调整配置: $(oput blue "SystemdCgroup = true")"
    sed -i "s/SystemdCgroup = false/SystemdCgroup = true/" "${etcContainerd}"

    echo "调整配置: $(oput blue "root = ${containerdDataDir}/lib")"
    sed -i "s|^root = \"/var/lib/containerd\"|root = \"${containerdDataDir}/lib\"|" "${etcContainerd}"
    echo "调整配置: $(oput blue "state = ${containerdDataDir}/run")"
    sed -i "s|^state = \"/run/containerd\"|state = \"${containerdDataDir}/run\"|" "${etcContainerd}"

    # 如果设置添加仓库认证信息则执行 注意不要修改以下if条件语句汇总的 sed 中的缩进
    if [ "${addRegAuth}" = "1" ]; then
        case "${criRuntime}" in
            "containerd" )
                # https://github.com/containerd/containerd/blob/main/docs/cri/registry.md#configure-registry-credentials
                # cri 为 containerd 容器运行时 的时候
                # echo "仓库认证: 添加 $(oput blue "${imgRegAddr}") 的认证信息到 $(oput blue "${etcContainerd}")"
#                sed -i '/\[plugins\."io.containerd.grpc.v1.cri"\.registry\.configs\]/a\
#        [plugins."io.containerd.grpc.v1.cri".registry.configs."'${imgRegAddr%%/*}'".tls]\
#          insecure_skip_verify = true\
#          ca_file = "'${regCaPath}'"\
#          cert_file = "'${regCrtPath}'"\
#          key_file = "'${regKeyPath}'"\
#        [plugins."io.containerd.grpc.v1.cri".registry.configs."'${imgRegAddr%%/*}'".auth]\
#          username = "'"${regUser}"'"\
#          password = "'"${regPass}"'"\
#          # auth = ""\
#          # identitytoken = ""' "${etcContainerd}"
#                sed -i '/\[plugins\."io.containerd.grpc.v1.cri"\.registry\.mirrors\]/a\
#        [plugin."io.containerd.grpc.v1.cri".registry.mirrors."'${imgRegAddr%%/*}'"]\
#          endpoint = ["'${regProtocol}'://'${imgRegAddr%%/*}'"]' "${etcContainerd}"

                # 镜像仓库证书保存目录不为空则复制到镜像仓库证书保存目录 containerd 的默认目录 /etc/containerd/certs.d
                # if [ -d "${certsDir}" ]; then cp -af "${certsDir}"/* "${certsDirContainerd}/"; fi

                # https://github.com/containerd/containerd/blob/main/docs/hosts.md
                # 此配置无需重启 containerd 即可生效
                echo "仓库认证: 添加 $(oput blue "${regAddr}") 的认证信息到 $(oput blue "${etcContainerd}")"
                if [ -d "${certsDir}" ]; then cp -af "${certsDir}"/* "${certsDirContainerd}/"; fi
                sed -i "/\[plugins.\"io.containerd.grpc.v1.cri\".registry\]/{n;s|config_path = \"\"|config_path = \"${certsDirContainerd}\"|}" "${etcContainerd}"
                local regCertDir="${certsDirContainerd}/${regAddr}"
                mkdir -p "${regCertDir}"
                cat > "${regCertDir}/hosts.toml" <<EOF
[host."https://${regAddr}"]
  capabilities = ["pull", "resolve", "push"]
  skip_verify = true
  tls:
    ca_file: "${regCaPath}"
    cert_file: "${regCrtPath}"
    key_file: "${regKeyPath}"
  auth:
    username: "${regUser}"
    password: "${regPass}"
EOF
                # 检查证书文件是否存在并给出提示
                local tlsList
                tlsList=("${regCaPath}" "${regCrtPath}" "${regKeyPath}")
                for tls in "${tlsList[@]}"; do
                    if [ ! -f "${tls}" ]; then
                        echo "证书文件: ${tls} $(oput red "未找到") 记得上传并重启 containerd 服务"
                    else
                        echo "证书文件: ${tls} $(oput green "已找到")"
                    fi
                done
                ;;
            "docker" )
                # cri 为 docker 容器运行时 的时候
                # 在 /etc/docker/daemon.json 直接添加 "insecure-registries": ["镜像仓库地址"] 即可
                ;;
        esac
    fi

    echo -e "创建文件: $(oput blue "${srvFileContainerd}")"
    cat > "${srvFileContainerd}" << EOF
# Copyright The containerd Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

[Unit]
Description=containerd container runtime
Documentation=https://containerd.io
After=network.target local-fs.target

[Service]
#uncomment to enable the experimental sbservice (sandboxed) version of containerd/cri integration
#Environment="ENABLE_CRI_SANDBOXES=sandboxed"
ExecStartPre=-/sbin/modprobe overlay
ExecStart=${installRootDir}/bin/containerd

Type=notify
Delegate=yes
KillMode=process
Restart=always
RestartSec=5
# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNPROC=infinity
LimitCORE=infinity
LimitNOFILE=infinity
# Comment TasksMax if your systemd version does not supports it.
# Only systemd 226 and above support this version.
TasksMax=infinity
OOMScoreAdjust=-999

[Install]
WantedBy=multi-user.target
EOF

    # cni-plugins
    echo -e "安装软件: $(oput blue "cni-plugins") \c"
    if tar -xf "${cniPluginsFile}" -C "${cniPluginsInstallDir}/bin"; then
        oput green "成功"
    else
        oput red "失败"; return 1
    fi

    # runc
    echo -e "安装软件: $(oput blue "runc") \c"
    if cp -af "${runcFile}" "${installRootDir}/sbin/runc"; then
        oput green "成功"
        chmod +x "${installRootDir}/sbin/runc"
    else
        oput red "失败"; return 1
    fi

    # cri-tools
    if ! install_crictl; then return 1; fi

    # 启动服务
    systemctl daemon-reload
    local srvSaveFile
    srvSaveFile="${docDir}/srv-cmd-containerd.txt"
    echo "创建文件: $(oput blue "${srvSaveFile}")"
    echo; generate_srv_cmd "containerd" "start restart stop status enable disable" | tee "${srvSaveFile}"

    srvList=("containerd")
    set_bootup "${srvList[@]}"

    echo; oput red "版本信息"
    echo; oput blue "containerd"
    containerd -version
    echo; oput blue "ctr"
    ctr version
    echo; oput blue "crictl"
    crictl --version
    echo; oput blue "CNI protocol versions supported"
    ${cniPluginsInstallDir}/bin/host-local
}

# 卸载 containerd 容器运行时
function remove_containerd() {
    oput red "卸载 containerd 容器运行时"; echo
    echo "暂不支持卸载"
}

# 安装 docker 容器运行时 - 预留
function install_docker() {
    oput red "安装 docker 容器运行时"
    echo; oput red "官方建议生产环境不要使用二进制文件的方式安装 所以采用在线安装 请确保可正常访问互联网"
    sleep 10
    echo

    appList=("docker"); srvList=("docker")
    if chk_install "${appList[@]}"; then
        case "${actOnExist}" in
            "1" )
                echo; echo "根据脚本参数设定 即将 $(oput red "覆盖安装") $(oput blue "${appList[*]}")"
                ;;
            "2" )
                echo; echo "根据脚本参数设定 即将 $(oput yellow "跳过安装") $(oput blue "${appList[*]}")"
                return 0
                ;;
            * )
                echo; echo "根据脚本参数设定 即将 $(oput blue "全新安装") $(oput blue "${appList[*]}")"
                ;;
        esac
    fi
    if chk_srv "${srvList[@]}"; then return 1; fi

    local CMD
    case "${osFamily}" in
        "redhat" )
            # 红帽系如果是欧拉、麒麟、UOS时 需要把 $releasever 改为 8
            # https://download.docker.com/linux/centos/docker-ce.repo
            yum remove docker docker-client docker-client-latest docker-common \
                docker-latest docker-latest-logrotate docker-logrotate docker-engine &>>"${sLog}"
            cat > /etc/yum.repos.d/docker-ce.repo <<'EOF'
[docker-ce-stable]
name=Docker CE Stable - $basearch
baseurl=https://download.docker.com/linux/centos/$releasever/$basearch/stable
enabled=1
gpgcheck=1
gpgkey=https://download.docker.com/linux/centos/gpg
EOF
#            cat > /etc/yum.repos.d/docker-ce.repo << 'EOF'
#[docker-ce-stable]
#name=Docker CE Stable - $basearch
#baseurl=https://repos.oso.plus/docker-ce/linux/centos/$releasever/$basearch/stable
## baseurl=https://repos.oso.plus/docker-ce/linux/centos/8/$basearch/stable
#enabled=1
#gpgcheck=1
#gpgkey=https://repos.oso.plus/docker-ce/linux/centos/gpg
#EOF
            iCMD="${osDCMD} install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin"
            # rCMD="${osDCMD} remove -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras"
            ;;
        "debian" )
            # apt -y update
            # apt -y install ca-certificates curl gnupg
            # install -m 0755 -d /etc/apt/keyrings
            # curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg
            # chmod a+r /etc/apt/keyrings/docker.gpg

            # Add the repository to Apt sources:
            # echo \
            # "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
            # $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
            # apt -y update

            for pkg in docker.io docker-doc docker-compose podman-docker containerd runc; do
                ${osDCMD} remove -y "${pkg}" &>>"${sLog}"
            done
            iCMD="${osDCMD} install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin"
            # rCMD="${osDCMD} remove -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin docker-ce-rootless-extras"
            ;;
    esac
    echo -e "安装软件: $(oput blue "docker") \c"
    if eval "${iCMD}" &>>"${sLog}"; then
        # echo "$(oput green "成功") 卸载命令: $(oput blue "${rCMD}")"
        oput green "成功"
    else
        oput red "失败"; return 1
    fi
    echo "export PATH=/usr/libexec/docker/cli-plugins:\$PATH" >> /etc/profile
    source /etc/profile

    if [ -f "${dockerDaemonFile}" ]; then cp "${dockerDaemonFile}" "${dockerDaemonFile}-$(date '+%Y%m%d%H%M%S')"; fi
    echo "创建文件: $(oput blue "${dockerDaemonFile}")"
    cat > "${dockerDaemonFile}" << EOF
{
    "insecure-registries": [
        "${imgRegAddr%/*}"
    ],
    "exec-opts": [
        "native.cgroupdriver=systemd"
    ],
    "registry-mirrors": [
        "https://${imgRegAddr%/*}",
        "https://registry.docker-cn.com",
        "https://hub-mirror.c.163.com",
        "https://docker.mirrors.ustc.edu.cn"
    ],
    "data-root": "${dockerDataDir}",
    "log-driver": "json-file",
    "log-opts": {
        "max-size": "100m"
    },
    "experimental": true
}
EOF

    # 保存服务管理命令
    systemctl daemon-reload
    local srvSaveFile
    srvSaveFile="${docDir}/srv-cmd-docker.txt"
    echo "创建文件: $(oput blue "${srvSaveFile}")"
    echo; generate_srv_cmd "docker" "start restart stop status enable disable" | tee "${srvSaveFile}"
    # 启动服务且加入开机自启
    if ! set_bootup "${srvList[@]}"; then return 1; fi

    # 安装 cri-tools
    if ! install_crictl; then return 1; fi

    # 报错: [ERROR CRI]: container runtime is not running: output: time="2023-12-22T01:20:15+08:00" level=fatal msg="validate service connection: validate CRI v1 runtime API for endpoint \"unix:///run/containerd/containerd.sock\": rpc error: code = Unimplemented desc = unknown service runtime.v1.RuntimeService"
    # 解决: 取消 disabled_plugins = ["cri"] 中的 "cri"
    if [ -f "${etcContainerd}" ]; then
        sed -i 's/disabled_plugins = \[\"cri\"\]/disabled_plugins = []/' "${etcContainerd}"
        systemctl restart containerd &>>"${sLog}"
    fi

    # 安装 cri-docker
    appList=("cri-docker"); srvList=("cri-docker")
    echo -e "安装软件: $(oput blue "cri-docker") \c"
    if tar -xf "${criDockerFile}" --strip-components=1 -C "${installRootDir}/bin"; then
        oput green "成功"
    else
        oput red "失败"; return 1
    fi

    echo "创建文件: $(oput blue "${etcCriDockerdSocket}")"
    cat > "${etcCriDockerdSocket}" << EOF
[Unit]
Description=CRI Docker Socket for the API
PartOf=cri-docker.service

[Socket]
ListenStream=%t/cri-dockerd.sock
SocketMode=0660
SocketUser=root
SocketGroup=docker

[Install]
WantedBy=sockets.target
EOF

    echo "创建文件: $(oput blue "${srvFileCriDockerd}")"
    cat > "${srvFileCriDockerd}" << EOF
[Unit]
Description=CRI Interface for Docker Application Container Engine
Documentation=https://docs.mirantis.com
After=network-online.target firewalld.service docker.service
Wants=network-online.target
Requires=cri-docker.socket

[Service]
Type=notify
ExecStart=${installRootDir}/bin/cri-dockerd --container-runtime-endpoint fd:// --pod-infra-container-image=${imgRegAddr}/pause:${pauseVer}
ExecReload=/bin/kill -s HUP \$MAINPID
TimeoutSec=0
RestartSec=2
Restart=always

# Note that StartLimit* options were moved from "Service" to "Unit" in systemd 229.
# Both the old, and new location are accepted by systemd 229 and up, so using the old location
# to make them work for either version of systemd.
StartLimitBurst=3

# Note that StartLimitInterval was renamed to StartLimitIntervalSec in systemd 230.
# Both the old, and new name are accepted by systemd 230 and up, so using the old name to make
# this option work for either version of systemd.
StartLimitInterval=60s

# Having non-zero Limit*s causes performance problems due to accounting overhead
# in the kernel. We recommend using cgroups to do container-local accounting.
LimitNOFILE=infinity
LimitNPROC=infinity
LimitCORE=infinity

# Comment TasksMax if your systemd version does not support it.
# Only systemd 226 and above support this option.
TasksMax=infinity
Delegate=yes
KillMode=process

[Install]
WantedBy=multi-user.target
EOF
    # 启动服务且加入开机自启
    if ! set_bootup "${srvList[@]}"; then return 1; fi

    echo; oput red "版本信息"
    echo; oput blue "docker"
    dockerd --version
    echo; oput blue "docker-buildx"
    docker-buildx version
    echo; oput blue "docker-compose"
    docker-compose --version
    echo; oput blue "cri-tools"
    crictl --version
    echo; oput blue "cri-docker"
    cri-dockerd --version
}

# 卸载 docker 容器运行时 - 预留
function remove_docker() {
    oput red "卸载 docker 容器运行时"; echo
    echo "暂不支持卸载 docker 容器运行时"
}

# 创建 k8s 集群初始化配置文件
function generate_k8s_config() {
    # oput red "创建 k8s 集群初始化配置文件"; echo
    cat << EOF > "${initFile}"
apiVersion: kubeadm.k8s.io/v1beta3
bootstrapTokens:
- groups:
  - system:bootstrappers:kubeadm:default-node-token
  token: abcdef.0123456789abcdef
  ttl: 24h0m0s
  usages:
  - signing
  - authentication
kind: InitConfiguration
localAPIEndpoint:
  advertiseAddress: $(echo "${k8sNodesMaster[0]}" | awk '{print $1}')
  bindPort: ${bindPort}
nodeRegistration:
  criSocket: ${criSockFile}
  imagePullPolicy: IfNotPresent
  name: $(echo "${k8sNodesMaster[0]}" | awk '{print $2}')
  taints: null
---
apiServer:
  timeoutForControlPlane: 4m0s
  certSANs:
  - ${vipAddr}
$(
for node in "${k8sNodesAll[@]}"; do
    echo "  - $(echo "${node}" | awk '{print $1}')"
    echo "  - $(echo "${node}" | awk '{print $2}')"
done
)
  - ${vipHostName}
  - ${clusterName}
  - ${vipDomain}
  - ${dnsDomain}
apiVersion: kubeadm.k8s.io/v1beta3
certificatesDir: /etc/kubernetes/pki
clusterName: ${clusterName}
controllerManager: {}
dns: {}
etcd:
  local:
    dataDir: /var/lib/etcd
imageRepository: ${imgRegAddr}
kind: ClusterConfiguration
kubernetesVersion: ${k8sVer}
networking:
  dnsDomain: ${dnsDomain}
  podSubnet: ${podSubnet}
  serviceSubnet: ${serviceSubnet}
scheduler: {}
controlPlaneEndpoint: "${vipAddr}:${vipPort}"
# controlPlaneEndpoint: "${vipDomain}:${vipPort}"
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs
---
apiVersion: kubelet.config.k8s.io/v1beta1
kind: KubeletConfiguration
cgroupDriver: systemd
EOF

    # 当 master 节点为 1 时
    if [ "${#k8sNodesMaster[@]}" -eq 1 ]; then
        # sed -i "s#^controlPlaneEndpoint.*#controlPlaneEndpoint: $(echo "${k8sNodesMaster[0]}" | awk '{print $1}'):${bindPort}#" "${initFile}"
        sed -i "/^  - ${vipHostName}/d" "${initFile}"
        sed -i "s#${vipAddr}#$(echo "${k8sNodesMaster[0]}" | awk '{print $1}')#g" "${initFile}"
        sed -i "s#${vipPort}#${bindPort}#g" "${initFile}"
    fi
    echo -e "创建文件: $(oput blue "${initFile}") \c"
    if [ -s "${initFile}" ]; then
        oput green "成功"
    else
        oput green "失败"
        return 1
    fi
}

# 安装 k8s 集群软件
function install_k8s() {
    if [ "${hostAddr}" = "$(echo "${k8sNodesMaster[0]}" | awk '{print $1}')" ]; then
        echo; echo "正在本地节点 $(oput blue "${hostAddr}") 上部署 $(oput blue "k8s集群相关软件")"; echo
    fi

    oput red "安装 k8s 集群软件"

    appList=("kubelet"); srvList=("kubelet")
    if chk_install "${appList[@]}"; then
        case "${actOnExist}" in
            "1" )
                echo; echo "根据脚本参数设定 即将 $(oput red "覆盖安装") $(oput blue "${appList[*]}")"
                ;;
            "2" )
                echo; echo "根据脚本参数设定 即将 $(oput yellow "跳过安装") $(oput blue "${appList[*]}")"
                return 0
                ;;
            * )
                echo; echo "根据脚本参数设定 即将 $(oput blue "全新安装") $(oput blue "${appList[*]}")"
                ;;
        esac
    fi
    if chk_srv "${srvList[@]}"; then return 1; fi

    # 检查实际版本和预期版本是否匹配
    local badApps=()
    local iList
    iList=("kubeadm" "kubectl" "kubelet")
    # echo; oput red "检查实际版本和预期版本是否匹配"
    for app in "${iList[@]}"; do
        case "${app}" in
            "kubeadm" )
                f="${kubeadmFile}"
                chmod +x "${f}"
                # actualVerKubeadm="$("${f}" version | grep -oP '(?<=GitVersion:")v?\K[^"]+')"
                actualVerKubeadm="$("${f}" version --output=yaml 2>/dev/null | grep -Ei 'gitVersion' | grep -Eo '[0-9.]+' | head -1)"
                actualVer="${actualVerKubeadm}"
                ;;
            "kubectl" )
                f="${kubectlFile}"
                chmod +x "${f}"
                actualVerKubectl="$("${f}" version --output=yaml 2>/dev/null | grep -Ei 'gitVersion' | grep -Eo '[0-9.]+' | head -1)"
                actualVer="${actualVerKubectl}"
                ;;
            "kubelet" )
                f="${kubeletFile}"
                chmod +x "${f}"
                actualVerKubelet="$("${f}" --version | grep -Eo '[0-9.]+')"
                actualVer="${actualVerKubelet}"
                ;;
        esac
        if [ "${actualVer}" != "${k8sVer}" ]; then badApps+=("${app} ${actualVer}"); fi
    done
    if [ "${#badApps[@]}" -gt 0 ]; then
        echo; oput red "版本检查未通过"
        for entry in "${badApps[@]}"; do
            app=$(echo "${entry}" | awk '{print $1}')
            actualVer=$(echo "${entry}" | awk '{print $2}')
            echo "$(oput blue "${app}") 的实际版本$(oput blue "(${actualVer})") $(oput red "!=") 预期版本$(oput blue "(${k8sVer})")"
        done
        return 1
    else
        # echo; oput green "版本检查已通过"
        echo
        for app in "${iList[@]}"; do
            case "${app}" in
                "kubeadm" ) f="${kubeadmFile}"; ;;
                "kubectl" ) f="${kubectlFile}"; ;;
                "kubelet" ) f="${kubeletFile}"; ;;
            esac
            echo -e "安装软件: $(oput blue "${app}") \c"
            if install -o root -g root -m 0755 "${f}" "${installRootDir}/bin/${app}"; then
                oput green "成功"
            else
                oput red "失败"; return 1
            fi
        done
    fi
    echo "创建文件: $(oput blue "${etcKubeadmConf}")"
    mkdir -p "/etc/systemd/system/kubelet.service.d"
    cat > "${etcKubeadmConf}" << EOF
# Note: This dropin only works with kubeadm and kubelet v1.11+
[Service]
Environment="KUBELET_KUBECONFIG_ARGS=--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
Environment="KUBELET_CONFIG_ARGS=--config=/var/lib/kubelet/config.yaml"
$(if [ ${kubeletDataDir} != "" ]; then echo "Environment=\"KUBELET_EXTRA_ARGS=--root-dir=${kubeletDataDir}/data\""; fi)
# This is a file that "kubeadm init" and "kubeadm join" generates at runtime, populating the KUBELET_KUBEADM_ARGS variable dynamically
EnvironmentFile=-/var/lib/kubelet/kubeadm-flags.env
# This is a file that the user can use for overrides of the kubelet args as a last resort. Preferably, the user should use
# the .NodeRegistration.KubeletExtraArgs object in the configuration files instead. KUBELET_EXTRA_ARGS should be sourced from this file.
EnvironmentFile=-/etc/sysconfig/kubelet
ExecStart=
ExecStart=${installRootDir}/bin/kubelet \$KUBELET_KUBECONFIG_ARGS \$KUBELET_CONFIG_ARGS \$KUBELET_KUBEADM_ARGS \$KUBELET_EXTRA_ARGS
EOF

    echo "创建文件: $(oput blue "${srvFileKubelet}")"
    cat > "${srvFileKubelet}" << EOF
[Unit]
Description=kubelet: The Kubernetes Node Agent
Documentation=https://kubernetes.io/docs/
Wants=network-online.target
After=network-online.target

[Service]
ExecStart=${installRootDir}/bin/kubelet
Restart=always
StartLimitInterval=0
RestartSec=10

[Install]
WantedBy=multi-user.target
EOF

    # 启动服务
    systemctl daemon-reload
    local srvSaveFile
    srvSaveFile="${docDir}/srv-cmd-kubelet.txt"
    echo "创建文件: $(oput blue "${srvSaveFile}")"

    echo; generate_srv_cmd "kubelet" "start restart stop status enable disable" | tee "${srvSaveFile}"

    set_bootup "${srvList[@]}"

    echo; oput red "版本信息"
    echo; oput blue "kubectl"
    kubectl version
    echo; oput blue "kubeadm"
    kubeadm version
    echo; oput blue "kubelet"
    kubelet --version
}

# 导入离线镜像
function imp_k8s_images() {
    # 保存 k8s 所需的离线镜像列表到变量中
    if ! generate_k8s_config; then return 1; fi
    local imgListArray
    if [ -f "${initFile}" ]; then
        readarray -t imgListArray < <(kubeadm config images list --config "${initFile}" | grep "${imgRegAddr}")
        imgListArray+=("${calicoRegAddr}/cni:v${calicoVer}")
        imgListArray+=("${calicoRegAddr}/kube-controllers:v${calicoVer}")
        imgListArray+=("${calicoRegAddr}/node:v${calicoVer}")
        if [ "${installMetricsServer}" = "1" ]; then
            # 由于模板文件允许部署在master和worker节点 所以注释掉了节点角色的判断
            # if printf "%s\n" "${k8sNodesMaster[@]}" | grep -q "${hostAddr}"; then
                imgListArray+=("${metricsServerRegAddr}/metrics-server:v${metricsServerVer}")
            # fi
        fi
    else
        oput red "文件不存在: ${initFile}"
        return 1
    fi

    local impCMD lsCMD rmCMD tagCMD
    case "${criRuntime}" in
        "containerd" )
            impCMD="ctr -n k8s.io i import"
            lsCMD="ctr -n k8s.io i list -q"
            rmCMD="ctr -n k8s.io i rm"
            tagCMD="ctr -n k8s.io i tag"
            ;;
        "docker" )
            impCMD="docker load -i"
            lsCMD="docker images -a"
            rmCMD="docker rmi -f"
            tagCMD="docker tag"
            ;;
    esac

    # 导入镜像
    local imgName imgFile imgListArray
    local i id=0
    for i in "${imgListArray[@]}"; do
        id=$((id+1))
        imgName="$(echo "${i##*/}" | awk -F ':' '{print $1}')"
        case "${imgName}" in
            "coredns" ) imgFile="${corednsFile}"; ;;
            "etcd" ) imgFile="${etcdFile}"; ;;
            "kube-apiserver" ) imgFile="${kubeApiFile}"; ;;
            "kube-proxy" ) imgFile="${kubeProxyFile}"; ;;
            "kube-controller-manager" ) imgFile="${kubeCMFile}"; ;;
            "kube-scheduler" ) imgFile="${kubeSchFile}"; ;;
            "pause" ) imgFile="${pauseFile}"; ;;
            "cni" ) imgFile="${calicoCniFile}"; ;;
            "node" ) imgFile="${calicoNodeFile}"; ;;
            "kube-controllers" ) imgFile="${calicoKCFile}"; ;;
            "metrics-server" ) imgFile="${metricsServerFile}"; ;;
            "metrics-scraper" ) imgFile="${metricsScraperFile}"; ;;
        esac
        echo -e "${id}/${#imgListArray[@]} 导入离线镜像: $(oput blue "${imgFile}") \c"
        if eval "${lsCMD}" | grep -q "${i}"; then
            oput yellow "跳过 镜像已存在"
            continue
        fi
        if eval "${impCMD}" "${imgFile}" >> "${sLog}" 2>&1; then
            oput green "成功"
        else
            oput red "失败 请检查镜像是否存在且是否支持当前硬件平台"
            return 1
        fi
    done

    echo
#    for i in "${imgListArray[@]}"; do
#        includeArchImg="${i/:/-${archType}:}"
#        if eval "${lsCMD}" | grep -q "^${i}"; then
#            # 如果存在 包含架构的镜像 且 存在 "${i}" 则删除包含架构的镜像 因为无需再次标记
#            if eval "${lsCMD}" | grep -q "^${includeArchImg}" && eval "${lsCMD}" | grep -q "^${i}"; then
#                echo -e "删除无用镜像: $(oput blue "${includeArchImg}") \c"
#                if eval "${rmCMD}" "${includeArchImg}" &>>"${sLog}" 2>&1; then oput green "成功"; else oput red "失败"; fi
#                continue
#            else
#                echo -e "重新标记镜像: $(oput blue "${includeArchImg}") to $(oput blue "${i}") \c"
#                if eval "${tagCMD}" "${includeArchImg}" "${i}" &>>"${sLog}" 2>&1; then
#                    oput green "成功"
#                    echo -e "删除无用镜像: $(oput blue "${includeArchImg}") \c"
#                    if eval "${rmCMD}" "${includeArchImg}" &>>"${sLog}" 2>&1; then oput green "成功"; else oput red "失败"; fi
#                else
#                    oput red "失败"
#                    return 1
#                fi
#            fi
#        else
#            continue
#        fi
#    done
    for i in "${imgListArray[@]}"; do
        includeArchImg="${i/:/-${archType}:}"

        # 使用一次命令获取所有镜像列表，并检查原始镜像和包含架构的镜像是否存在
        local images
        images="$(eval "${lsCMD}")"
        if [[ "${images}" == *"$i"* ]]; then
            if [[ "${images}" == *"${includeArchImg}"* ]]; then
                # 如果两个镜像都存在，则删除包含架构的镜像
                echo -e "删除无用镜像: $(oput blue "${includeArchImg}") \c"
                if eval "${rmCMD}" "${includeArchImg}" &>>"${sLog}" 2>&1; then
                    oput green "成功"
                else
                    oput red "失败"
                    return 1
                fi
            else
                # 判断 包含架构的镜像是否存在 如果存在则重新标记
                if eval "${lsCMD}" | grep -q "^${includeArchImg}"; then
                    # 只有原始镜像存在，则重新标记并删除包含架构的镜像
                    echo -e "重新标记镜像: $(oput blue "${includeArchImg}") to $(oput blue "${i}") \c"
                    if eval "${tagCMD}" "${includeArchImg}" "${i}" &>>"${sLog}" 2>&1; then
                        oput green "成功"
                        echo -e "删除无用镜像: $(oput blue "${includeArchImg}") \c"
                        if eval "${rmCMD}" "${includeArchImg}" &>>"${sLog}" 2>&1; then
                            oput green "成功"
                        else
                            oput red "失败"
                            return 1
                        fi
                    else
                        oput red "失败"
                        return 1
                    fi
                else
                    continue
                fi
            fi
        fi
    done
}

# k8s集群初始化成功后 添加 kube 环境变量 仅 master 节点执行
function add_kubectl_env() {
    echo; oput red "配置环境变量"; echo
    echo -e "创建目录: $(oput blue "${HOME}"/.kube) \c"
    if mkdir -p "${HOME}"/.kube; then oput green "成功"; else oput red "失败"; fi
    echo -e "复制文件: $(oput blue "/etc/kubernetes/admin.conf") to $(oput blue "${HOME}"/.kube/config) \c"
    if cp -af /etc/kubernetes/admin.conf "${HOME}"/.kube/config; then oput green "成功"; else oput red "失败"; fi
    echo -e "配置属主: $(oput blue "${HOME}"/.kube/config) to $(oput blue "$(id -u):$(id -g)") \c"
    if chown "$(id -u):$(id -g)" "${HOME}"/.kube/config; then oput green "成功"; else oput red "失败"; fi
    # export KUBECONFIG=${HOME}/.kube/config
    echo -e "环境变量: $(oput blue "export KUBECONFIG=\$HOME/.kube/config") \c"
    if grep -q "^export KUBECONFIG=\$HOME/.kube/config" "${HOME}"/.bashrc; then
        oput green "无需再次添加"
    else
        if echo "export KUBECONFIG=\$HOME/.kube/config" >> "${HOME}"/.bashrc; then
            oput green "添加成功"
            # shellcheck source=/dev/null
            source "${HOME}"/.bashrc
        else
            oput red "添加失败"
        fi
    fi
    if [ "${addAlias}" = "1" ]; then
        if [ -f ~/.bashrc ]; then
            for x in "${aliasContent[@]}"; do
                echo -e "别名变量: $(oput blue "${x}") \c"
                if grep -q "${x}" "${HOME}"/.bashrc; then
                    oput yellow "无需再次添加"
                else
                    if echo "${x}" >> "${HOME}"/.bashrc; then oput green "添加成功"; else oput red "添加失败"; fi
                fi
            done
            source ~/.bashrc
            # echo "环境变量: added kubectl's command alias to the '~/.bashrc'"
        fi
    fi
    local envVar="export PATH=${installRootDir}/bin:${installRootDir}/sbin:\$PATH"
    echo -e "环境变量: $(oput blue "${envVar}") \c"
    if grep -q "${envVar}" /etc/profile; then
        oput yellow "无需再次添加"
    else
        if echo "${envVar}" >> /etc/profile && source /etc/profile; then
            oput green "添加成功"
            # shellcheck source=/dev/null
            source /etc/profile
        else
            oput red "添加失败"
        fi
    fi
}

# 初始化 k8s 集群
function init_k8s_cluster() {
    oput red "初始化 k8s 集群"
    if [ "$(echo "${k8sNodesMaster[0]}" | awk '{print $1}')" != "${hostAddr}" ]; then
        echo; echo "请在 master 第一节点: $(oput blue "${k8sNodesMaster[0]}") 上执行初始化操作"
        return 1
    fi

    # 创建初始化配置文件
    echo; if ! generate_k8s_config; then return 1; fi

    # 检查 kubelet 服务是否正在运行 如果正在运行则关闭 避免报错 [ERROR Port-10250]: Port 10250 is in use
    if chk_srv "kubelet"; then return 1; fi

    # master 节点大于 1 时 才需要检查 vip 的漂移情况
    if [ "${#k8sNodesMaster[@]}" -gt 1 ]; then
        # 检查 vip 是否在 master 第一节点上 上
        # 关闭其余节点上的 master 节点上的 keepalived 服务 确认已经漂移到 master 第一节点后 再启动服务
        echo; oput red "VIP 漂移检查"
        if ! hostname -I | grep -q "${vipAddr}"; then
            echo; echo "vip 地址 $(oput blue "${vipAddr}") 在当前节点 $(oput blue "${hostAddr}") 上 $(oput red "未找到") 正尝试漂移切换"
            # ${k8sNodesMaster[@]:1} 会取出数组 k8sNodesMaster 中除了第一个元素外的所有元素
            for node in "${k8sNodesMaster[@]:1}"; do
                echo -e "$(oput blue "关闭") 远程节点 $(echo "${node}" | awk '{print $1}') 上的 $(oput blue "keepalived") 服务 \c"
                CMD="gossh -t cmd -h $(echo "${node}" | awk '{print $1}') ${gOpts} \"systemctl stop keepalived\""
                if eval "${CMD}" | grep -q 'return=0'; then oput green "成功"; else oput red "失败"; return 1; fi
            done
            # 再次检查
            if ! hostname -I | grep -q "${vipAddr}"; then
                echo "vip 地址 ${vipAddr} 未能漂移到当前节点 $(oput blue "${hostAddr}") 请解决后再次运行脚本"
                return 1
            else
                echo
                echo "vip 地址 $(oput blue "${vipAddr}") 在当前节点 $(oput blue "${hostAddr}") 上 $(oput green "已找到")"
                for node in "${k8sNodesMaster[@]:1}"; do
                    echo -e "$(oput blue "启动") 远程节点 $(echo "${node}" | awk '{print $1}') 上的 $(oput blue "keepalived") 服务 \c"
                    CMD="gossh -t cmd -h $(echo "${node}" | awk '{print $1}') ${gOpts} \"systemctl start keepalived\""
                    if eval "${CMD}" | grep -q 'return=0'; then oput green "成功"; else oput red "失败"; return 1; fi
                done
            fi
        else
            echo; echo "vip 地址 $(oput blue "${vipAddr}") 在当前节点 $(oput blue "${hostAddr}") 上 $(oput green "已找到")"
        fi
    fi

    # 初始化集群之前先 在线拉取镜像 或 导入离线镜像
    case "${imagePolicy}" in
        "1" )
            echo; oput red "正在拉取镜像"
            if kubeadm config images pull --config "${initFile}"; then
                oput green "镜像拉取成功"
            else
                oput red "镜像拉取失败 请检查网络和镜像仓库是否正常"
                return 1
            fi
            ;;
        "2" )
            echo; oput red "导入离线镜像"
            echo; if ! imp_k8s_images; then return 1; fi
            # 远程节点上导入离线镜像
            id=0; ok=0; no=0
            for x in "${k8sNodesAll[@]:1}"; do
                id=$((id+1))
                tNode="$(echo "${x}" | awk '{print $1}')"
                # 注意: $((${#k8sNodesAll[@]}-1)) 的写法必须确保元素个数大于1
                echo -e "${id}/$((${#k8sNodesAll[@]}-1)) 正在为远程节点 $(oput blue "${x}") 导入离线镜像 \c"
                CMD="gossh -t cmd -h ${tNode} ${gOpts} \"cd ${pushToDir} && bash ${sFile} l &>> ${sLog}\""
                if eval "${CMD}" | tee -a "${sLog}" | grep -q 'return=0' >/dev/null 2>&1; then
                    ok=$((ok+1))
                    oput green "成功"
                else
                    no=$((no+1))
                    oput red "失败"
                fi
            done
            echo; echo "远程节点上导入离线镜像 成功 $(oput green "${ok}") 个 失败 $(oput red "${no}") 个"
            ;;
    esac

    # 执行集群初始化
    touch "${initLog}" && true >"${initLog}"

    # 重置集群信息
    { kubeadm reset -f; ipvsadm --clear; rm -rf ~/.kube; } &>/dev/null
    # 检查集群初始化是否成功的查找字符串
    grepStr="Your Kubernetes control-plane has initialized successfully"

    echo; oput red "初始化集群"
    initCMD="kubeadm init --upload-certs --config ${initFile}"
    echo; echo -e "执行命令: $(oput blue "${initCMD}") \c"
    # if kubeadm init --upload-certs --config "${initFile}" | tee /dev/stderr | grep -q "${grepStr}"; then
    if eval "${initCMD}" &>>"${initLog}"; then oput green "成功"; else oput red "失败"; return 1; fi
    echo -e "检查结果: \c"
    if grep -q "${grepStr}" "${initLog}"; then
        oput green "初始化成功"
        # 添加 kube 相关配置环境变量
        add_kubectl_env
    else
        echo; oput red "集群初始化失败 执行以下命令查看日志 ${initLog}"
        echo; oput red "再次初始化之前请先重置集群信息"
        echo "重置集群: $(oput blue "kubeadm reset -f; ipvsadm --clear; rm -rf ~/.kube")"
        echo "拉取镜像: $(oput blue "kubeadm config images pull --config ${initFile}")"
        echo "init集群: $(oput blue "kubeadm init --upload-certs --config ${initFile}")"
        return 1
    fi
}

# 加入节点到 k8s 集群
function join_k8s_nodes() {
    oput red "集群节点加入"

    if [ ! -f "${initLog}" ]; then echo; oput red "获取节点 join 命令失败 未找到文件 ${initLog}"; return 1; fi

    # 清理节点加入日志命令
    cLogCMD="touch ${joinLog} && true > ${joinLog}"
    # k8s 集群重置命令
    reCMD="kubeadm reset -f; ipvsadm --clear; rm -rf ~/.kube"
    # 使用 docker 时加入节点需要的附加参数
    if [ "${criRuntime}" = "docker" ]; then args="--cri-socket ${criSockFile}"; fi

    echo
    # 从日志中获取 master/worker 节点加入命令
    idRole=0
    local roles=("master" "worker")
    for role in "${roles[@]}"; do
        idRole=$((idRole+1))
        case "${role}" in
            "master" )
                kw="You can now join any number of the control-plane node"
                num=5
                ;;
            "worker" )
                kw="Then you can join any number of worker nodes"
                num=4
                ;;
        esac
        joinCMD="$(grep -A"${num}" "${kw}" "${initLog}" | sed "/${kw}/d" | sed -e '/^$/d; s/^[ \t]*//; s/[ \t]*$//')"
        echo -e "获取 $(oput red "${role}") 节点 join 命令 \c"
        if [ -z "${joinCMD}" ]; then oput red "失败"; return 1; else oput green "成功"; fi
        # echo -e "join命令: \n${joinCMD}"

        # 远程节点执行节点加入命令处理逻辑
        local id=0; local ok=0; local no=0;
        function process_node() {
            id=$((id+1))
            ip="$(echo "${x}" | awk '{print $1}')"
            case "${role}" in
                "master" )
                    progStat="${idRole}/${#roles[@]}/${id}/$((${#k8sNodesMaster[@]}-1))/$((${#k8sNodesAll[@]}-1))"
                    ;;
                "worker" )
                    progStat="${idRole}/${#roles[@]}/${id}/${#k8sNodesWorker[@]}/$((${#k8sNodesAll[@]}-1))"
                    ;;
            esac

            # 先重置节点 避免重复安装时报错
            echo -e "${progStat} 正在远程将 $(oput blue "${role}") 节点 $(oput blue "${x}") 加入集群 \c"
            CMD="gossh -t cmd -h ${ip} ${gOpts} \"${cLogCMD}; ${reCMD} &>> ${sLog}; ${joinCMD} ${args} &>> ${joinLog}\""
            if eval "${CMD}" | tee -a "${joinLog}" | grep -q 'return=0'; then
                ok=$((ok+1))
                oput green "成功"
                # 添加环境变量
                if [ "${role}" = "master" ]; then
                    echo -e "${progStat} 正在远程为 $(oput blue "${role}") 节点 $(oput blue "${x}") 添加环境变量 \c"
                    CMD="gossh -t cmd -h ${ip} ${gOpts} \"cd ${pushToDir} && bash ${sFile} e\""
                    if eval "${CMD}" | tee -a "${sLog}" | grep -q 'return=0'; then oput green "成功";  else oput red "失败"; fi
                    # if eval "${CMD}" | grep -q 'return=0'; then oput green "成功";  else oput red "失败"; fi
                fi
            else
                no=$((no+1))
                oput red "失败"
                echo "请使用以下命令手动尝试"
                oput blue "
kubeadm reset -f; ipvsadm --clear; rm -rf ~/.kube
ipvsadm --clear
rm -rf \$HOME/.kube/config
rm -rf /etc/cni/net.d/*
"
                oput blue "${joinCMD}"; echo
            fi
        }

        case "${role}" in
            "master" ) for x in "${k8sNodesMaster[@]:1}"; do process_node "${x}"; done; ;;
            "worker" ) for x in "${k8sNodesWorker[@]}"; do process_node "${x}"; done; ;;
        esac
        # todo 单 master 节点 的集群时不需要这个提示
        echo; echo "加入 $(oput blue "${role}") 节点 成功 $(oput green "${ok}") 个 失败 $(oput red "${no}") 个"; echo
    done

    oput red "当前集群节点信息"
    kubectl get nodes
    return 0
}

# 卸载 k8s 集群软件
function remove_k8s() {
    oput red "卸载 k8s 集群软件"; echo
    oput red "暂不支持自动卸载 请参考以下命令手动卸载"
    echo; oput blue "# 高可用相关软件卸载"
            cat <<EOF
systemctl stop keepalived haproxy
${osDCMD} remove -y keepalived haproxy
rm -rf /etc/{keepalived,haproxy}
EOF

    echo; oput blue "# 重置并清理节点信息"
            cat <<EOF
kubeadm reset -f; ipvsadm --clear; rm -rf ~/.kube
EOF

    case "${criRuntime}" in
        "docker" )
            echo; oput blue "# 基于 docker 容器运行时的卸载"
            cat <<EOF
while read x; do ctr -n=k8s.io i rm "\${x}"; done <<< "\$(docker images -a | grep -v 'k8s.gcr.io' | awk '{print \$3}')"

systemctl stop kubelet docker cri-docker
rm -rf /usr/local/{bin,sbin}/*
rm -rf /etc/kubernetes/
rm -rf /etc/docker/
rm -rf /etc/crictl.yaml
rm -rf /etc/containerd/
rm -f /etc/systemd/system/containerd.service
rm -rf /etc/cni
rm -rf /opt/{cni,containerd}
rm -rf /var/lib/etcd
rm -rf ~/.kube/
EOF
            ;;
        "containerd" )
            echo; oput blue "# 基于 containerd 容器运行时的卸载"
            cat <<EOF
while read x; do ctr -n=k8s.io i rm "\${x}"; done <<< "\$(ctr -n=k8s.io i ls -q)"

systemctl stop kubelet containerd
rm -rf /usr/local/sbin/runc
rm -rf /usr/local/bin/{containerd,containerd-shim,containerd-shim-runc-v1,containerd-shim-runc-v2,containerd-stress,crictl,ctr,kubeadm,kubectl,kubelet}
rm -rf /etc/kubernetes/
rm -rf /etc/crictl.yaml
rm -rf /etc/containerd/
rm -f /etc/systemd/system/containerd.service
rm -rf /etc/cni
rm -rf /opt/{cni,containerd}
rm -rf /var/lib/etcd
rm -rf ~/.kube/
EOF
            ;;
        * )
            oput red "未知的容器运行时 ${criRuntime}"
            ;;
    esac
}

# 安装 calico 网络插件
function install_calico() {
    oput red "安装 calico 网络插件"; echo

    # 网络设置
    if systemctl status NetworkManager >/dev/null 2>>"${sLog}"; then
        cat > "${calicoNetConf}" << 'EOF'
[keyfile]
unmanaged-devices=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:vxlan-v6.calico;interface-name:wireguard.cali;interface-name:wg-v6.cali
EOF
        systemctl restart NetworkManager >/dev/null 2>>"${sLog}"
    fi

    echo "复制文件: $(oput blue "${sDir}/template/calico-${calicoVer}.yaml") to $(oput blue "calico.yaml")"
    if ! cp -af "${sDir}/template/calico-${calicoVer}.yaml" "calico.yaml"; then
        echo; oput red "复制 calico 模板文件失败 ${sDir}/template/calico-${calicoVer}.yaml"; return 1
    fi

    echo "设置文件: $(oput blue "calico.yaml")"
    sed -i "s?10.244.0.0/16?${podSubnet}?" "calico.yaml"
    sed -i "s?interface=ens33?interface=${ethName}?" "calico.yaml"
    sed -i "s|image: .*/calico|image: ${calicoRegAddr}|g" "calico.yaml"
    if [ "${calicoDisableIPIP}" = "1" ]; then
        sed -i '/- name: CALICO_IPV4POOL_IPIP/{N;s/value: "Always"/value: "Never"/}' "calico.yaml"
    fi

    echo -e "应用文件: $(oput blue "calico.yaml") \c"
    if kubectl apply -f "calico.yaml" >>"${sLog}" 2>&1; then oput green "成功"; else oput red "失败"; return 1; fi
    echo; echo "请参考以下命令检查 $(oput blue "calico") 是否部署成功"
    cat <<EOF
$(oput blue "kubectl get nodes")
$(oput blue "kubectl get pod -A")
$(oput blue "kubectl get pod -n kube-system")
EOF
}

# 安装 metrics-server 或 metrics-scraper
function install_metrics_server() {
    oput red "部署 metrics-server 组件"; echo

    echo "复制文件: $(oput blue "${templateFilemetricsServer}") to $(oput blue "metrics-server.yaml")"
    if ! cp -af "${templateFilemetricsServer}" "metrics-server.yaml"; then
        echo; oput red "复制 metrics-server 模板文件失败"; return 1
    fi
    echo "设置文件: $(oput blue "metrics-server.yaml")"
    sed -i "s|image: .*/calico|image: ${metricsServerRegAddr}|g" "metrics-server.yaml"
    echo -e "应用文件: $(oput blue "metrics-server.yaml") \c"
    if kubectl apply -f "metrics-server.yaml" >>"${sLog}" 2>&1; then oput green "成功"; else oput red "失败"; return 1; fi

    echo; echo "请参考以下命令检查 $(oput blue "metrics-server") 是否部署成功"
    cat <<EOF
$(oput blue "kubectl get deploy -n kube-system")
$(oput blue "kubectl get pod -n kube-system")
$(oput blue "kubectl -n kube-system top pod")
$(oput blue "kubectl top nodes")
EOF
}

# 安装高可用和负载均衡软件 包含 keepalived 和 haproxy
function install_ha() {
    oput red "安装高可用和负载均衡软件"

    appList=("keepalived" "haproxy"); srvList=("keepalived" "haproxy")
    if chk_install "${appList[@]}"; then
        case "${actOnExist}" in
            "1" )
                echo; echo "根据脚本参数设定 即将 $(oput red "覆盖安装") $(oput blue "${appList[*]}")"
                ;;
            "2" )
                echo; echo "根据脚本参数设定 即将 $(oput yellow "跳过安装") $(oput blue "${appList[*]}")"
                return 0
                ;;
            * )
                echo; echo "根据脚本参数设定 即将 $(oput blue "全新安装") $(oput blue "${appList[*]}")"
                ;;
        esac
    fi
    if chk_srv "${srvList[@]}"; then return 1; fi

    echo
    echo -e "安装软件: $(oput blue "keepalived") 和 $(oput blue "haproxy") \c"
    local CMD
    CMD="${osDCMD} install -y keepalived haproxy"
    if eval "${CMD}" &>>"${sLog}"; then oput green "成功"; else oput red "失败"; return 1; fi

    echo "创建文件: $(oput blue "${etcFileKeepalived}")"
    if [ -f "${etcFileKeepalived}" ]; then mv "${etcFileKeepalived}" "${etcFileKeepalived}-$(date +%Y%m%d%H%M%S)"; fi
    cat > "${etcFileKeepalived}" << EOF
! Configuration File for keepalived
global_defs {
  notification_email {
  }
  router_id LVS_DEVEL
  vrrp_skip_check_adv_addr
  vrrp_garp_interval 0
  vrrp_gna_interval 0
}
vrrp_script chk_haproxy {
  script "/usr/bin/pgrep haproxy"
  interval 2
  timeout 1
  weight 2
}
vrrp_instance k8s-vip {
  state BACKUP
  priority 100
  interface ${ethName} # Network card
  virtual_router_id ${vipAddr##*.}
  advert_int 1
  authentication {
    auth_type PASS
    auth_pass myAuth-0
  }
  unicast_src_ip ${hostAddr} # The IP address of this machine
  unicast_peer {
$(for w in "${k8sNodesMaster[@]}"; do
    if [ "${hostAddr}" != "$(echo "${w}" | awk '{print $1}')" ]; then
        echo "    $(echo "${w}" | awk '{print $1}') # master node $(echo "${w}" | awk '{print $2}')'s IP address"
    fi
done)
  }
  virtual_ipaddress {
    ${vipAddr}/24 # The VIP address
  }
  track_script {
    chk_haproxy
  }
}
EOF

    echo "创建文件: $(oput blue "${etcFileHaproxy}")"
    if [ -f "${etcFileHaproxy}" ]; then mv "${etcFileHaproxy}" "${etcFileHaproxy}-$(date +%Y%m%d%H%M%S)"; fi
    cat > "${etcFileHaproxy}" << EOF
global
    log /dev/log  local0 warning
    chroot      /var/lib/haproxy
    pidfile     /var/run/haproxy.pid
    maxconn     4000
    user        haproxy
    group       haproxy
    daemon
    stats socket /var/lib/haproxy/stats
defaults
    log global
    option  httplog
    option  dontlognull
        timeout connect 5000
        timeout client 50000
        timeout server 50000
frontend kube-apiserver
    bind *:${vipPort}
    mode tcp
    option tcplog
    default_backend kube-apiserver
backend kube-apiserver
    mode tcp
    option tcp-check
    balance roundrobin
    default-server inter 10s downinter 5s rise 2 fall 2 slowstart 60s maxconn 250 maxqueue 256 weight 100
$(id=0;for z in "${k8sNodesMaster[@]}"; do
    id=$((id+1))
    echo "    server kube-apiserver-${id} $(echo "${z}" | awk '{print $1}'):${bindPort} check"
done)
EOF

    systemctl daemon-reload
    local srvSaveFile
    srvSaveFile="${docDir}/srv-cmd-ha.txt"
    echo "创建文件: $(oput blue "${srvSaveFile}")"
    echo; generate_srv_cmd "keepalived haproxy" "start restart stop status enable disable" | tee "${srvSaveFile}"

    set_bootup "${srvList[@]}"
}

# 卸载高可用和负载均衡软件 包含 keepalived 和 haproxy
function remove_ha() {
    oput red "卸载高可用和负载均衡软件"; echo
    local CMD
    CMD="${osDCMD} remove -y keepalived haproxy"
    echo -e "正在卸载软件: $(oput blue "keepalived") 和 $(oput blue "haproxy") \c"
    if eval "${CMD}" >> "${sLog}" 2>&1; then
        echo "$(oput green "成功") 请手动清理 $(oput blue "/etc/{keepalived,haproxy}") 下的配置文件"
    else
        oput red "失败"
        return 1
    fi
    systemctl daemon-reload
}

# 生成服务管理命令
function generate_srv_cmd() {
    local -a srvList
    local -a srvActions

    # 使用 read 命令将参数按照空格分割为数组
    read -ra srvList <<< "$1"
    read -ra srvActions <<< "$2"

    for srv in "${srvList[@]}"; do
        echo "服务管理命令 - ${srv}"
        for action in "${srvActions[@]}"; do
            case "${action}" in
                "start") msg="启动服务命令"; ;;
                "restart") msg="重启服务命令"; ;;
                "stop") msg="关闭服务命令"; ;;
                "status") msg="查看服务状态"; ;;
                "enable") msg="启用开机自启"; ;;
                "disable") msg="取消开机自启"; ;;
            esac
            echo "${msg}: systemctl ${action} ${srv}"
        done
        echo ""
    done
}

# 部署完成后执行额外任务
function extras_tasks (){
    if [ "${addSecret}" = "1" ]; then
        if ! kubectl get namespace ${secretNS} &>/dev/null; then kubectl create namespace ${secretNS} &>/dev/null; fi
            createSecretCMD="kubectl create secret -n ${secretNS} docker-registry ${secretName} \
            --docker-username=${dockerUser} --docker-password=${dockerPass} --docker-server=${dockerServer}"
        echo; echo -e "创建 ${secretName} secret: \c"
        if eval "${createSecretCMD}" &>>"${sLog}"; then oput green "成功"; else oput red "失败"; fi
    fi
}

function msg_para_info() {
    oput green "k8s 集群 配置信息"
    echo "vip 主机名: $(oput blue "${vipHostName}")"
    echo "vip 地址: $(oput blue "${vipAddr}")"
    echo "vip 端口: $(oput blue "${vipPort}")"
    echo "vip 域名: $(oput blue "${vipDomain}")"
    echo "容器运行时: $(oput blue "${criRuntime}")"
    echo "cri 套接字: $(oput blue "${criSockFile}")"
    echo "镜像仓库: $(oput blue "${imgRegAddr}")"
    echo "k8s 版本: $(oput blue "${k8sVer}")"
    # echo "沙箱版本: $(oput blue "${pauseVer}")"

    echo "集群名称: $(oput blue "${clusterName}")"
    echo "dns 域名: $(oput blue "${dnsDomain}")"
    echo "pod 子网: $(oput blue "${podSubnet}")"
    echo "服务子网: $(oput blue "${serviceSubnet}")"
    echo "localAPIEndpoint: $(oput blue "$(echo "${k8sNodesMaster[0]}" | awk '{print $1}'):${bindPort}")"
    local id
    id=1
    for x in "${k8sNodesMaster[@]}"; do
        echo "master 节点${id}: $(oput blue "$x")"
        id=$((id+1))
    done
    id=1
    for x in "${k8sNodesWorker[@]}"; do
        echo "worker 节点${id}: $(oput blue "$x")"
        id=$((id+1))
    done
    id=0
    echo; oput green "k8s 主机 连接信息"
    echo "服务器端口: $(oput blue "${sshPort}")"
    echo "服务器用户名: $(oput blue "${sshUser}")"
    echo "服务器密码: $(oput blue "${sshPWD}")"

    echo; oput green "calico 配置信息"
    echo "版本: $(oput blue "${calicoVer}")"
    echo "仓库: $(oput blue "${calicoRegAddr}")"
    # echo "网卡: $(oput blue "${calicoIpAutodetectionMethodValue}")"
    echo "子网: $(oput blue "${podSubnet}")"
    # echo "子网: $(oput blue "${calicoIpv4poolCidrValue}")"

    echo; oput green "镜像仓库 配置信息"
    case "${imagePolicy}" in
        "1" )
            echo "镜像策略: $(oput blue "在线拉取")"
            ;;
        "2" )
            echo "镜像策略: $(oput blue "离线导入")"
            ;;
    esac
    echo "登录用户: $(oput blue "${regUser}")"
    echo "登录密码: $(oput blue "${regPass}")"
    echo "访问协议: $(oput blue "${regProtocol}")"
    echo "CA 证书: $(oput blue "${regCaPath}")"
    echo "Cert证书: $(oput blue "${regCrtPath}")"
    echo "证书私钥: $(oput blue "${regKeyPath}")"

    echo; oput green "其他配置信息"
    echo "数据目录: containerd $(oput blue "${containerdDataDir}")"
    echo "数据目录: kubelet $(oput blue "${kubeletDataDir}")"
    echo "数据目录: etcd $(oput blue "${etcdDataDir}")"
    if [ "${addSecret}" = "1" ]; then echo "添加秘密: secret $(oput blue "是")"; fi
    if [ "${addAlias}" = "1" ]; then echo "添加别名: kubectl $(oput blue "是")"; fi
    if [ "${installHelm}" = "1" ]; then echo "附加安装: helm $(oput blue "是")"; fi
    if [ "${installMetricsServer}" = "1" ]; then echo "附加安装: metrics-server $(oput blue "是")"; fi
}

function script_info() {
    # echo "脚本目录: $(oput blue "${sDir}")"
    echo "脚本文件: $(oput blue "${sFile}")"
    echo "脚本功能: $(oput blue "${sDesc}")"
    echo "支持环境: $(oput blue "${sEnv}")"
    echo "脚本用法: $(oput blue "${sUsage}")"
    echo "脚本日志: $(oput blue "${sLogDir}")"
    echo "创建时间: $(oput blue "${sCreate}")"
    echo "更新时间: $(oput blue "${sUpdate}")"
    echo
    echo "登录信息: $(oput blue "$(id -un)@${hostName}(${hostAddr}) from ${fromAddr}")"
    echo "平台架构: $(oput blue "${archType}")"
    echo "CPU 型号: $(oput blue "${cpuModel}")"
    echo "硬件概览: $(oput blue "${cpuCount} 核CPU + ${memSize} GB内存 + ${diskSize:=0} GB磁盘")"
    echo "操作系统: $(oput blue "${osRelease}")"
    echo "系统内核: $(oput blue "${osKernel}")"
    echo "Bash版本: $(oput blue "${BASH_VERSION}")"
    echo "Libc版本: $(oput blue "${osLibcVer}")"
}

# 脚本帮助信息
function usage_details() {
    oput red "脚本帮助信息"
    cat << EOF

$(oput green "环境支持")
红帽家族: $(oput blue "RHEL/Alma/Rocky/OL/Anolis/CentOS 7/8/9 | UOS V20 1060a/1060e | Kylin V10 | OpenEuler 22.03")
德班家族: $(oput blue "Debian 11/12 | Ubuntu 20.04/22.04")
集群支持: $(oput blue "containerd|docker + k8s 1.2[4-9].x/1.30.x + calico 3.2[678].x")
节点支持: $(oput blue "奇数个 master 节点 + 任意个 worker 节点")

$(oput green "脚本用法")
安装准备: $(oput blue "上传 脚本/安装文件/镜像仓库证书到第一个主节点上")
参数修改: $(oput blue "设置 脚本内的相关参数并保存")
文件分发: $(oput blue "执行 bash ${sFile} p 分发文件到所有集群节点")
集群部署: $(oput blue "执行 bash ${sFile} i 部署 k8s集群和相关软件")
传参示例: $(oput blue "${sUsage}")
EOF
}

# 文件分发
function push_files() {
    echo; oput red "文件分发"

    # 需要推送的文件和目录的汇总清单 imagePolicy 1=在线拉取 2=离线导入
    toPushList=(
        "${sDir}/${sFile}"
        "${sBinDir}"
        "${certsDir}"
        "${softwareDir}"
        "${templateDir}"
    )
    case "${imagePolicy}" in
        "1" )
            ;;
        "2" )
            toPushList+=(
                "${imgDir}"
            )
            ;;
    esac

    local id=0
    for x in "${k8sNodesAll[@]:1}"; do
        id=$((id+1))
        targetIP="$(echo "${x}" | awk '{print $1}')"
        targetName="$(echo "${x}" | awk '{print $2}')"
        echo; oput blue "${id}/$((${#k8sNodesAll[@]}-1)) ${targetIP} ${targetName}"

        CMD="gossh -t cmd -h ${targetIP} ${gOpts} \"mkdir -p \"${sLogDir}\" && touch \"${sLog}\" && true >\"${sLog}\"; mkdir -p ${pushToDir}/{certs,bin,software,template,images}\""
        # eval "${CMD}"
        echo -e "创建目录: $(oput blue "${targetIP}:$(echo "${CMD}" | awk '{print $(NF)}' | sed 's/"//g')") \c"
        if eval "${CMD}" | grep -q 'return=0'; then oput green "成功"; else oput red "失败"; return 1; fi
        for obj in "${toPushList[@]}"; do
            case "${obj}" in
                "${certsDir}" )
                    CMD="gossh -t push -h ${targetIP} ${gOpts} \"${obj}\" \"${pushToDir}\""
                    ;;
                "${sBinDir}" )
                    CMD="gossh -t push -h ${targetIP} ${gOpts} \"${obj}\" \"${pushToDir}/bin\""
                    ;;
                "${softwareDir}" )
                    CMD="gossh -t push -h ${targetIP} ${gOpts} \"${obj}\" \"${pushToDir}/software\""
                    ;;
                "${templateDir}" )
                    CMD="gossh -t push -h ${targetIP} ${gOpts} \"${obj}\" \"${pushToDir}\""
                    ;;
                "${sDir}/${sFile}" )
                    CMD="gossh -t push -h ${targetIP} ${gOpts} \"${obj}\" \"${pushToDir}\""
                    ;;
                "${imgDir}" )
                    CMD="gossh -t push -h ${targetIP} ${gOpts} \"${obj}\" \"${pushToDir}/images\""
                    ;;
                # "${helmFile}" )
                #     CMD="gossh -t push -h ${targetIP} ${gOpts} \"${obj}\" \"${pushToDir}\""
                #     ;;
            esac
            echo -e "推送文件: $(oput blue "${obj}") to $(oput blue "${targetIP}:$(echo "${CMD}" | awk '{print $(NF-0)}' | sed 's/"//g')") \c"
            if eval "${CMD}" | grep -q 'return=0'; then oput green "成功"; else oput red "失败"; return 1; fi
        done
    done
}

function ask_continue() {
    local msg
    case "${action}" in
        "i" | "install" )
            msg="部署基于 ${criRuntime} 的 k8s 集群"
            ;;
        "r" | "remove" )
            msg="卸载基于 ${criRuntime} 的 k8s 集群"
            ;;
        "p" | "push" )
            msg="分发 k8s 集群部署文件到所有集群节点"
            ;;
    esac
    echo
    read -rp "即将 $(oput blue "${msg}") 是否继续? [y/N] " yn
    echo
    case "${yn}" in
        [yY] ) return 0; ;;
        * ) return 1; ;;
    esac
}

# 公共信息
function public_msg() {
    # 输出脚本配置信息
    echo; script_info
    echo; msg_para_info
    echo; usage_details
}

# 执行安装
function exec_install() {
    # 安装容器运行时
    case "${criRuntime}" in
        "containerd" ) echo; if ! install_containerd; then echo; return 1; fi; ;;
        "docker" ) echo; if ! install_docker; then echo; return 1; fi; ;;
    esac

    # 安装高可用组件 - 仅 master 节点上执行
    if [ "${#k8sNodesMaster[@]}" -gt 1 ]; then
        for node in "${k8sNodesMaster[@]}"; do
            if [ "$(echo "${node}" | awk '{print $1}')" = "${hostAddr}" ]; then
                echo; if ! install_ha; then echo; return 1; fi
                if [ "${installHelm}" = "1" ]; then
                    :
                fi
            fi
        done
    fi

    # 安装k8s集群件
    echo; if ! install_k8s; then echo; return 1; fi

    # 安装helm软件
    for node in "${k8sNodesMaster[@]}"; do
        if [ "$(echo "${node}" | awk '{print $1}')" = "${hostAddr}" ] && [ "${installHelm}" = "1" ]; then
            echo; echo -e "安装helm: \c"
            if tar -xf "${helmFile}" --strip-components=1 -C "${installRootDir}/bin" linux-${archType}/helm; then
                oput green "成功 已装版本 $("${installRootDir}/bin/helm" version 2>&1 | grep -Eo 'v3.[0-9.]+')"
            else
                oput red "失败"
                return 1
            fi
        fi
    done

    # 当前脚本运行于 master 第一个节点时
    case "${hostAddr}" in
        "$(echo "${k8sNodesMaster[0]}" | awk '{print $1}')" )
            echo; oput red "在远程节点上安装k8s集群相关软件"; echo
            local id=0
            for x in "${k8sNodesAll[@]:1}"; do
                id=$((id+1))
                ip="$(echo "${x}" | awk '{print $1}')"
                echo -e "${id}/$((${#k8sNodesAll[@]}-1)) 正在远程节点 $(oput blue "${x}") 上部署 $(oput blue "k8s集群相关软件") \c"
                remoteCMD="export ethName=${ethName}; cd ${pushToDir} && bash ${sFile} i"
                CMD="gossh -t cmd -h ${ip} ${gOpts} \"${remoteCMD}\""
                if eval "${CMD}" | grep -q 'return=0'; then oput green "成功"; else oput red "失败"; return 1; fi
            done

            # 执行k8s集群初始化
            echo; if ! init_k8s_cluster; then echo; return 1; fi

            # 加入集群节点
            echo; if ! join_k8s_nodes; then echo; return 1; fi

            # 安装 calico 网络组件
            echo; if ! install_calico; then echo; return 1; fi

            # 安装 metrics-server 组件
            if [ "${installMetricsServer}" = "1" ]; then echo; if ! install_metrics_server; then echo; return 1; fi; fi

            # 安装完成后执行额外任务
            extras_tasks

            # 打印安装成功信息
            echo; oput green "k8s 集群已安装完成且成功启动"
            ;;
    esac
}

# 获取当前时间
get_current_time() { date +"%Y-%m-%d %H:%M:%S"; }
# 计算时间差
# 函数调用方法: time_diff "sleep 5"
time_diff() {
    local start_time
    start_time=$(get_current_time)
    eval "$1"
    local end_time
    end_time=$(get_current_time)
    local diff_seconds=$(($(date -d "${end_time}" +%s) - $(date -d "${start_time}" +%s)))
    local days=$((diff_seconds / 86400))
    local hours=$(((diff_seconds % 86400) / 3600))
    local minutes=$(((diff_seconds % 3600) / 60))
    local seconds=$((diff_seconds % 60))
    echo
    echo "开始时间：$(oput blue "${start_time}")"
    echo "结束时间：$(oput blue "${end_time}")"
    echo "总计耗时：$(oput blue "${days}天${hours}时${minutes}分${seconds}秒")"
}

# 加载配置文件定义
def_etc_file
# 传参动作 只接受 i/install, r/remove, p/push, v/version
action="$1"
case "${action}" in
    "i" | "install" | "p" | "push" | "s" )
        # 获取所有节点信息
        if ! fetch_nodes; then exit 1; fi
        ;;
esac
case "${action}" in
    "i" | "install" )
        # 安装软件
        public_msg
        # m1 时选择是否继续
        case "${hostAddr}" in
            "$(echo "${k8sNodesMaster[0]}" | awk '{print $1}')" )
                # 询问是否继续 - master 1 节点
                if ! ask_continue; then exit 1; fi
                ;;
            * )
                ;;
        esac
        # 校验脚本预设参数
        echo; if ! chk_params; then echo; exit 1; fi
        # 检查文件
        echo; if ! find_files; then echo; exit 1; fi
        # 检查依赖
        echo; if ! chk_depend; then echo; exit 1; fi
        # 检查命令
        echo; if ! chk_command; then echo; exit 1; fi
        # 如果目标主机的IP地址为 master 1 节点的 IP 地址时 需要执行的操作
        if [ "${hostAddr}" = "$(echo "${k8sNodesMaster[0]}" | awk '{print $1}')" ]; then
                # 获取 master 第一节点的关联网卡名称 将作为 haproxy/keepalived/calico 等组件的网卡名称
                if ! get_eth_name "${hostAddr}"; then
                    oput red "获取 k8s 集群 控制平面节点 $(echo "${k8sNodesMaster[0]}" | awk '{print $1}') 的关联网卡名称失败"
                    exit 1
                fi
                # 连通性检查
                echo; if ! chk_ssh_conn; then echo; exit 1; fi
                # 确保已设置免密登录
                if [ "${setNoPwdLogin}" = "1" ]; then echo; set_nopass; fi
        fi
        # 优化系统
        echo; if ! tune_system; then echo; return 1; fi
        # 执行安装
        echo; if ! time_diff exec_install; then echo; exit 1; else echo; exit 0; fi
        ;;
    "r" | "remove" )
        # 卸载软件
        public_msg
        echo; remove_k8s
        ;;
    "p" | "push" )
        # 打印消息
        public_msg
        # 尝试解包相关部署文件
        echo; if [ "${tryUnPKG}" = "1" ]; then if ! try_unpack_pkg; then exit 1; fi; fi
        # m1 时选择是否继续
        case "${hostAddr}" in
            "$(echo "${k8sNodesMaster[0]}" | awk '{print $1}')" )
                # 询问是否继续 - master 1 节点
                if ! ask_continue; then exit 1; fi
                ;;
            * )
                ;;
        esac
        # 校验脚本预设参数
        echo; if ! chk_params; then echo; exit 1; fi
        # 检查文件
        echo; if ! find_files; then echo; exit 1; fi
        # 检查命令
        echo; if ! chk_command; then echo; exit 1; fi

        # 连通性检查
        echo; if ! chk_ssh_conn; then echo; exit 1; fi
        # 推送文件
        echo; if ! time_diff push_files; then echo; exit 1; else echo; exit 0; fi
        ;;
    "e" | "env" )
        # 添加 kube 环境变量 远程调用时使用
        # echo "当前主机 ${hostAddr}"
        add_kubectl_env
        echo; exit 0
        ;;
    "l" | "load" )
        # 导入离线镜像 远程调用时使用
        echo; if ! find_files; then echo; exit 1; fi
        echo; if ! imp_k8s_images; then echo; exit 1; fi
        ;;
    "s" )
        # 连接检查 测试使用
        echo; if ! chk_command; then echo; exit 1; fi
        echo; if ! chk_ssh_conn; then echo; exit 1; fi
        ;;
    "v" | "version" )
        # 版本信息
        echo; echo "script version: $(oput blue "${sUpdate}")"; echo; exit 0
        ;;
    * )
        echo; script_info
        echo; msg_para_info
        echo; usage_details
        echo; exit 0
        ;;
esac
